ci: add agent-scan workflow to flag bot-authored PRs

Author: danielroe

.github/workflows/agent-scan.yml

@@ -0,0 +1,115 @@
+name: agent-scan
+
+on:
+  # zizmor: ignore[dangerous-triggers] - DO NOT add action/checkout in this workflow as it uses pull_request_target
+  pull_request_target:
+    types:
+      - opened
+      - reopened
+
+concurrency:
+  group: agent-scan-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  agentscan:
+    runs-on: ubuntu-latest
+    steps:
+      - name: AgentScan
+        id: agentscan
+        uses: MatteoGabriele/agentscan-action@ccb50da057e2ac7fcdb123db3110f00af97f0fef # v1.14.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          skip-members: "dependabot[bot],renovate[bot]"
+          agent-scan-comment: false
+      - name: Handle flagged PR
+        if: contains(fromJSON('["automation","mixed"]'), steps.agentscan.outputs.classification) || steps.agentscan.outputs.community-flagged == 'true'
+        env:
+          CLASSIFICATION: ${{ steps.agentscan.outputs.classification }}
+          COMMUNITY_FLAGGED: ${{ steps.agentscan.outputs.community-flagged }}
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          script: |
+            const prNumber = context.payload.pull_request.number;
+            const classification = process.env.CLASSIFICATION;
+            const communityFlagged = process.env.COMMUNITY_FLAGGED === 'true';
+            const shouldClose = classification === 'automation' || communityFlagged;
+
+            const issue = context.payload.pull_request
+            const labels = issue.labels?.map(l => l.name) || []
+
+            if (!labels.includes('possible bot')) {
+              await github.rest.issues.addLabels({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                labels: ['possible bot'],
+              })
+            }
+
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+              per_page: 100,
+            })
+
+            const alreadyCommented = comments.some(
+              c => c.user.type === 'Bot' && c.body.includes('AI-assisted contribution guidelines')
+            )
+
+            if (!alreadyCommented) {
+              const closingNote = shouldClose
+                ? "We're closing this for now as the account looks automated. If we got that wrong, please just reopen the PR and we'll take another look."
+                : 'If this was flagged in error, we apologise! 😳 Just let us know. 🙏'
+
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                body: [
+                  "We've flagged this as a potential contribution without a human behind it. We welcome the thoughtful use of AI tools when contributing, but ask all contributors to follow [two core principles](https://roe.dev/blog/using-ai-in-open-source):",
+                  '',
+                  '1. **Never let an LLM speak for you** - all comments, issues, and PR descriptions should be written in your own words, reflecting your own understanding.',
+                  '2. **Never let an LLM think for you** - only submit contributions you fully understand and can explain.',
+                  '',
+                  'Please review these AI-assisted contribution guidelines and update this contribution if needed.',
+                  '',
+                  closingNote,
+                ].join('\n'),
+              })
+            } else {
+              core.info('Possible-bot comment already exists - skipping comment.')
+            }
+
+            if (shouldClose && issue.state === 'open' && !alreadyCommented) {
+              await github.rest.pulls.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                pull_number: prNumber,
+                state: 'closed',
+                title: '🚨 unwelcome pr from bot 🚨',
+              })
+            }
+
+            const actionTaken = [
+              'Added `possible bot` label',
+              alreadyCommented ? null : 'posted policy comment',
+              shouldClose && !alreadyCommented ? 'closed PR' : null,
+            ].filter(Boolean).join(', ')
+
+            core.summary
+              .addHeading('AgentScan: Possible Bot Flag', 2)
+              .addTable([
+                [{ data: 'Property', header: true }, { data: 'Value', header: true }],
+                ['Pull Request', `#${prNumber}`],
+                ['Classification', classification],
+                ['Community flagged', String(communityFlagged)],
+                ['Action', actionTaken || 'No action (already handled)'],
+              ])
+
+            await core.summary.write()