From 5af30bf862f19e8a9a7d6e419f690f62c65ad181 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Mon, 8 Jun 2026 12:46:53 +0200 Subject: [PATCH 1/2] ci: auto-close incomplete external PRs Add a pull_request_target workflow that checks PRs from external contributors (org members and bots are exempt via author_association): the PR template must be filled in, and any PR touching functional code (SDK, API, or frontend) must include a demo recording. Non-compliant PRs get a sticky comment, an incomplete-pr label, and are closed, then auto-reopened once fixed. Adds workflow_dispatch dry_run/force_external inputs for testing and registers the new label in .labels.yml. --- .github/.labels.yml | 3 + .../workflows/13-check-pr-contribution.yml | 206 ++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 .github/workflows/13-check-pr-contribution.yml diff --git a/.github/.labels.yml b/.github/.labels.yml index eda5ec87ee..4e5be4a136 100644 --- a/.github/.labels.yml +++ b/.github/.labels.yml @@ -109,6 +109,9 @@ - name: invalid description: This doesn't seem right color: e4e669 +- name: incomplete-pr + description: PR is missing required template sections or a demo recording + color: e11d21 - name: performance description: A code change that improves performance color: f3ffb2 diff --git a/.github/workflows/13-check-pr-contribution.yml b/.github/workflows/13-check-pr-contribution.yml new file mode 100644 index 0000000000..c649f7414d --- /dev/null +++ b/.github/workflows/13-check-pr-contribution.yml @@ -0,0 +1,206 @@ +name: "13 - check PR contribution" + +# Enforces the contribution requirements on pull requests opened by external +# contributors: the PR template must be filled in, and any PR that changes +# functional code (SDK, API, or frontend) must include a demo recording or +# screenshot. PRs that do not comply are commented on, labelled, and closed. +# +# Uses pull_request_target so the workflow has a write token even for fork PRs. +# It only reads PR metadata (body and changed-file list) and posts a comment. +# It never checks out or runs the PR's code, so this is safe. + +on: + pull_request_target: + types: [opened, edited, synchronize, reopened, ready_for_review] + workflow_dispatch: + inputs: + pr_number: + description: "PR number to evaluate" + required: true + type: string + dry_run: + description: "Only log the decision; do not comment, label, close, or reopen" + required: false + default: true + type: boolean + force_external: + description: "Ignore the org-membership exemption (treat the author as external)" + required: false + default: false + type: boolean + +permissions: + contents: read + pull-requests: write + issues: write + +concurrency: + group: pr-contribution-${{ github.event.pull_request.number || inputs.pr_number }} + cancel-in-progress: true + +jobs: + check: + name: Check contribution requirements + runs-on: ubuntu-latest + steps: + - name: Evaluate PR + uses: actions/github-script@v7 + with: + script: | + const MARKER = ''; + const LABEL = 'incomplete-pr'; + const INTERNAL = ['OWNER', 'MEMBER', 'COLLABORATOR']; + + // Files that are non-functional. A PR touching only these may skip the demo. + const EXEMPT = [ + /(^|\/)tests?\//i, + /(^|\/)__tests__\//i, + /(^|\/)test_[^/]*\.py$/i, + /_test\.py$/i, + /\.(test|spec)\.[jt]sx?$/i, + /^docs\//i, + /\.mdx?$/i, + /^\.github\//i, + /(^|\/)(package-lock\.json|pnpm-lock\.yaml|yarn\.lock|poetry\.lock|uv\.lock|Cargo\.lock)$/i, + /\.(png|jpe?g|gif|svg|webp|ico)$/i, + ]; + + // What counts as a demo in the Demo section. + const MEDIA = [ + /!\[[^\]]*\]\([^)]+\)/, // markdown image + //g, '').trim() : null; + } + + const reasons = []; + + // 1) Template is present and filled. + const headers = ['Summary', 'Testing', 'Demo', 'Checklist']; + const lower = body.toLowerCase(); + const missing = headers.filter((h) => !lower.includes('## ' + h.toLowerCase())); + if (!body.trim()) { + reasons.push('The pull request description is empty. Please fill in the PR template.'); + } else if (missing.length) { + reasons.push('The description is missing required sections (' + missing.join(', ') + '). Please use the PR template without removing its sections.'); + } else if (!section('Summary')) { + reasons.push('The **Summary** section is empty. Describe what changed and why.'); + } + + // 2) Demo is present for functional changes. + const files = await github.paginate(github.rest.pulls.listFiles, { + owner, repo, pull_number: number, per_page: 100, + }); + const functional = files.some((f) => !EXEMPT.some((r) => r.test(f.filename))); + const demo = section('Demo') || ''; + const hasMedia = MEDIA.some((r) => r.test(demo)); + if (functional && !hasMedia) { + reasons.push('This PR changes functional code (SDK, API, or frontend) but the **Demo** section has no screenshot or video. A short demo recording is required. Only test-only, docs-only, or chore changes may mark Demo as N/A.'); + } + + async function upsertComment(text) { + const comments = await github.paginate(github.rest.issues.listComments, { + owner, repo, issue_number: number, per_page: 100, + }); + const existing = comments.find((c) => c.body && c.body.includes(MARKER)); + if (existing) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body: text }); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number: number, body: text }); + } + } + + if (reasons.length) { + const list = reasons.map((r) => '- ' + r).join('\n'); + const text = [ + MARKER, + `Hi @${pr.user.login}, thanks for opening a pull request. šŸ™`, + '', + 'This PR was **automatically closed** because it does not yet meet our contribution requirements:', + '', + list, + '', + 'We ask for this so every change is documented and demonstrably tested before review.', + '', + '**How to get it reopened**', + 'Update the PR description (and add a demo recording if your change touches functional code). The bot reopens the PR automatically once the requirements are met. No need to open a new one.', + '', + 'See the [Contributing guide](https://agenta.ai/docs/contributing/overview) and [Creating your first PR](https://agenta.ai/docs/contributing/first-pr). If you think this was closed in error, leave a comment and a maintainer will take a look.', + ].join('\n'); + + if (dryRun) { + core.notice(`[dry-run] Would close PR #${number}:\n${list}`); + return; + } + + await upsertComment(text); + try { + await github.rest.issues.addLabels({ owner, repo, issue_number: number, labels: [LABEL] }); + } catch (e) { + core.warning(`Could not add label ${LABEL}: ${e.message}`); + } + if (pr.state === 'open') { + await github.rest.pulls.update({ owner, repo, pull_number: number, state: 'closed' }); + } + core.notice(`Closed PR #${number}:\n${list}`); + } else { + if (dryRun) { + core.info(`[dry-run] PR #${number} meets contribution requirements.`); + return; + } + // Compliant. If the bot had previously closed it, reopen and clear the flag. + const labels = (pr.labels || []).map((l) => l.name); + if (labels.includes(LABEL)) { + try { + await github.rest.issues.removeLabel({ owner, repo, issue_number: number, name: LABEL }); + } catch (e) { + core.warning(`Could not remove label ${LABEL}: ${e.message}`); + } + if (pr.state === 'closed') { + await github.rest.pulls.update({ owner, repo, pull_number: number, state: 'open' }); + } + await upsertComment(`${MARKER}\nāœ… Thanks @${pr.user.login}! This PR now meets the contribution requirements and has been reopened. A maintainer will review it soon.`); + } + core.info(`PR #${number} meets contribution requirements.`); + } From a2dc2f3f0dd6b4c88efda45b0e4e8c385317e707 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Mon, 8 Jun 2026 13:56:33 +0200 Subject: [PATCH 2/2] ci: stop PR bot over-closing on missing checklist + reopen loop Two fixes found while auditing the bot against all open PRs: - Only a non-empty Summary plus a demo (for functional changes) are required. Missing Testing/Checklist sections no longer close a PR. The demo is now detected anywhere in the body, not just the Demo section. This fixes a PR that had a YouTube demo and full testing notes but was closed for lacking the checklist section. - Drop the 'reopened' trigger so a maintainer who manually reopens a flagged PR wins, instead of the bot immediately re-closing it. Auto-reopen on a fixed description still works via 'edited'/'synchronize'. --- .../workflows/13-check-pr-contribution.yml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/13-check-pr-contribution.yml b/.github/workflows/13-check-pr-contribution.yml index c649f7414d..dfd9714b70 100644 --- a/.github/workflows/13-check-pr-contribution.yml +++ b/.github/workflows/13-check-pr-contribution.yml @@ -10,8 +10,11 @@ name: "13 - check PR contribution" # It never checks out or runs the PR's code, so this is safe. on: + # No 'reopened': a maintainer who manually reopens a flagged PR should win, + # otherwise the reopen event would immediately re-close it. Auto-reopen on a + # fixed description still works through 'edited' and 'synchronize'. pull_request_target: - types: [opened, edited, synchronize, reopened, ready_for_review] + types: [opened, edited, synchronize, ready_for_review] workflow_dispatch: inputs: pr_number: @@ -116,27 +119,24 @@ jobs: const reasons = []; - // 1) Template is present and filled. - const headers = ['Summary', 'Testing', 'Demo', 'Checklist']; - const lower = body.toLowerCase(); - const missing = headers.filter((h) => !lower.includes('## ' + h.toLowerCase())); + // 1) The PR is described. We only require a non-empty Summary, not the + // full template. Missing Testing/Checklist sections do not close a PR; + // a thorough PR with a demo should never be closed over a checklist. if (!body.trim()) { reasons.push('The pull request description is empty. Please fill in the PR template.'); - } else if (missing.length) { - reasons.push('The description is missing required sections (' + missing.join(', ') + '). Please use the PR template without removing its sections.'); } else if (!section('Summary')) { - reasons.push('The **Summary** section is empty. Describe what changed and why.'); + reasons.push('The **Summary** section is missing or empty. Describe what changed and why using the PR template.'); } - // 2) Demo is present for functional changes. + // 2) Demo is present for functional changes. Scan the whole body, not + // just the Demo section, so a screenshot or video placed anywhere counts. const files = await github.paginate(github.rest.pulls.listFiles, { owner, repo, pull_number: number, per_page: 100, }); const functional = files.some((f) => !EXEMPT.some((r) => r.test(f.filename))); - const demo = section('Demo') || ''; - const hasMedia = MEDIA.some((r) => r.test(demo)); + const hasMedia = MEDIA.some((r) => r.test(body)); if (functional && !hasMedia) { - reasons.push('This PR changes functional code (SDK, API, or frontend) but the **Demo** section has no screenshot or video. A short demo recording is required. Only test-only, docs-only, or chore changes may mark Demo as N/A.'); + reasons.push('This PR changes functional code (SDK, API, or frontend) but includes no demo. Add a screenshot or short video of the change. Only test-only, docs-only, or chore changes may skip it.'); } async function upsertComment(text) {