ci: stop PR bot over-closing on missing checklist + fix reopen loop

.github/.labels.yml

@@ -109,6 +109,9 @@
 - name: invalid
   description: This doesn't seem right
   color: e4e669
+- name: incomplete-pr
+  description: PR is missing required template sections or a demo recording
+  color: e11d21
 - name: performance
   description: A code change that improves performance
   color: f3ffb2

.github/workflows/13-check-pr-contribution.yml

@@ -0,0 +1,206 @@
+name: "13 - check PR contribution"
+
+# Enforces the contribution requirements on pull requests opened by external
+# contributors: the PR template must be filled in, and any PR that changes
+# functional code (SDK, API, or frontend) must include a demo recording or
+# screenshot. PRs that do not comply are commented on, labelled, and closed.
+#
+# Uses pull_request_target so the workflow has a write token even for fork PRs.
+# It only reads PR metadata (body and changed-file list) and posts a comment.
+# It never checks out or runs the PR's code, so this is safe.
+
+on:
+  # No 'reopened': a maintainer who manually reopens a flagged PR should win,
+  # otherwise the reopen event would immediately re-close it. Auto-reopen on a
+  # fixed description still works through 'edited' and 'synchronize'.
+  pull_request_target:
+    types: [opened, edited, synchronize, ready_for_review]
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: "PR number to evaluate"
+        required: true
+        type: string
+      dry_run:
+        description: "Only log the decision; do not comment, label, close, or reopen"
+        required: false
+        default: true
+        type: boolean
+      force_external:
+        description: "Ignore the org-membership exemption (treat the author as external)"
+        required: false
+        default: false
+        type: boolean
+
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+
+concurrency:
+  group: pr-contribution-${{ github.event.pull_request.number || inputs.pr_number }}
+  cancel-in-progress: true
+
+jobs:
+  check:
+    name: Check contribution requirements
+    runs-on: ubuntu-latest
+    steps:
+      - name: Evaluate PR
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const MARKER = '<!-- agenta-pr-contribution-check -->';
+            const LABEL = 'incomplete-pr';
+            const INTERNAL = ['OWNER', 'MEMBER', 'COLLABORATOR'];
+
+            // Files that are non-functional. A PR touching only these may skip the demo.
+            const EXEMPT = [
+              /(^|\/)tests?\//i,
+              /(^|\/)__tests__\//i,
+              /(^|\/)test_[^/]*\.py$/i,
+              /_test\.py$/i,
+              /\.(test|spec)\.[jt]sx?$/i,
+              /^docs\//i,
+              /\.mdx?$/i,
+              /^\.github\//i,
+              /(^|\/)(package-lock\.json|pnpm-lock\.yaml|yarn\.lock|poetry\.lock|uv\.lock|Cargo\.lock)$/i,
+              /\.(png|jpe?g|gif|svg|webp|ico)$/i,
+            ];
+
+            // What counts as a demo in the Demo section.
+            const MEDIA = [
+              /!\[[^\]]*\]\([^)]+\)/,                                  // markdown image
+              /<img\s/i,
+              /<video\s/i,
+              /https?:\/\/[^\s)]+\.(mp4|mov|webm|gif)/i,               // direct video/gif
+              /https?:\/\/(www\.)?(youtube\.com|youtu\.be)\//i,
+              /https?:\/\/(www\.)?loom\.com\//i,
+              /https?:\/\/[^\s)]*\/user-attachments\//i,               // GitHub uploads
+              /https?:\/\/[^\s)]*githubusercontent\.com\//i,
+            ];
+
+            const owner = context.repo.owner;
+            const repo = context.repo.repo;
+            const dispatch = context.eventName === 'workflow_dispatch';
+            const inputs = context.payload.inputs || {};
+            const dryRun = dispatch && String(inputs.dry_run) === 'true';
+            const forceExternal = dispatch && String(inputs.force_external) === 'true';
+
+            // Resolve the PR from the event payload, or fetch it for manual dispatch.
+            let pr;
+            if (dispatch) {
+              const num = Number(inputs.pr_number);
+              pr = (await github.rest.pulls.get({ owner, repo, pull_number: num })).data;
+            } else {
+              pr = context.payload.pull_request;
+            }
+            const number = pr.number;
+
+            // Drafts are work in progress; only enforce on ready PRs (or manual runs).
+            if (!dispatch && pr.draft) {
+              core.info(`PR #${number} is a draft, skipping.`);
+              return;
+            }
+
+            // Exempt internal contributors (org members) and bots.
+            if (!forceExternal && (pr.user.type === 'Bot' || INTERNAL.includes(pr.author_association))) {
+              core.info(`PR #${number} by ${pr.user.login} (${pr.author_association}) is internal, skipping.`);
+              return;
+            }
+
+            const body = pr.body || '';
+
+            function section(name) {
+              const re = new RegExp('##\\s*' + name + '\\b([\\s\\S]*?)(?=\\n##\\s|$)', 'i');
+              const m = body.match(re);
+              return m ? m[1].replace(/<!--[\s\S]*?-->/g, '').trim() : null;
+            }
+
+            const reasons = [];
+
+            // 1) The PR is described. We only require a non-empty Summary, not the
+            //    full template. Missing Testing/Checklist sections do not close a PR;
+            //    a thorough PR with a demo should never be closed over a checklist.
+            if (!body.trim()) {
+              reasons.push('The pull request description is empty. Please fill in the PR template.');
+            } else if (!section('Summary')) {
+              reasons.push('The **Summary** section is missing or empty. Describe what changed and why using the PR template.');
+            }
+
+            // 2) Demo is present for functional changes. Scan the whole body, not
+            //    just the Demo section, so a screenshot or video placed anywhere counts.
+            const files = await github.paginate(github.rest.pulls.listFiles, {
+              owner, repo, pull_number: number, per_page: 100,
+            });
+            const functional = files.some((f) => !EXEMPT.some((r) => r.test(f.filename)));
+            const hasMedia = MEDIA.some((r) => r.test(body));
+            if (functional && !hasMedia) {
+              reasons.push('This PR changes functional code (SDK, API, or frontend) but includes no demo. Add a screenshot or short video of the change. Only test-only, docs-only, or chore changes may skip it.');
+            }
+
+            async function upsertComment(text) {
+              const comments = await github.paginate(github.rest.issues.listComments, {
+                owner, repo, issue_number: number, per_page: 100,
+              });
+              const existing = comments.find((c) => c.body && c.body.includes(MARKER));
+              if (existing) {
+                await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body: text });
+              } else {
+                await github.rest.issues.createComment({ owner, repo, issue_number: number, body: text });
+              }
+            }
+
+            if (reasons.length) {
+              const list = reasons.map((r) => '- ' + r).join('\n');
+              const text = [
+                MARKER,
+                `Hi @${pr.user.login}, thanks for opening a pull request. 🙏`,
+                '',
+                'This PR was **automatically closed** because it does not yet meet our contribution requirements:',
+                '',
+                list,
+                '',
+                'We ask for this so every change is documented and demonstrably tested before review.',
+                '',
+                '**How to get it reopened**',
+                'Update the PR description (and add a demo recording if your change touches functional code). The bot reopens the PR automatically once the requirements are met. No need to open a new one.',
+                '',
+                'See the [Contributing guide](https://agenta.ai/docs/contributing/overview) and [Creating your first PR](https://agenta.ai/docs/contributing/first-pr). If you think this was closed in error, leave a comment and a maintainer will take a look.',
+              ].join('\n');
+
+              if (dryRun) {
+                core.notice(`[dry-run] Would close PR #${number}:\n${list}`);
+                return;
+              }
+
+              await upsertComment(text);
+              try {
+                await github.rest.issues.addLabels({ owner, repo, issue_number: number, labels: [LABEL] });
+              } catch (e) {
+                core.warning(`Could not add label ${LABEL}: ${e.message}`);
+              }
+              if (pr.state === 'open') {
+                await github.rest.pulls.update({ owner, repo, pull_number: number, state: 'closed' });
+              }
+              core.notice(`Closed PR #${number}:\n${list}`);
+            } else {
+              if (dryRun) {
+                core.info(`[dry-run] PR #${number} meets contribution requirements.`);
+                return;
+              }
+              // Compliant. If the bot had previously closed it, reopen and clear the flag.
+              const labels = (pr.labels || []).map((l) => l.name);
+              if (labels.includes(LABEL)) {
+                try {
+                  await github.rest.issues.removeLabel({ owner, repo, issue_number: number, name: LABEL });
+                } catch (e) {
+                  core.warning(`Could not remove label ${LABEL}: ${e.message}`);
+                }
+                if (pr.state === 'closed') {
+                  await github.rest.pulls.update({ owner, repo, pull_number: number, state: 'open' });
+                }
+                await upsertComment(`${MARKER}\n✅ Thanks @${pr.user.login}! This PR now meets the contribution requirements and has been reopened. A maintainer will review it soon.`);
+              }
+              core.info(`PR #${number} meets contribution requirements.`);
+            }