Skip to content

Commit b2348f8

Browse files
BagToadBagToad
authored
Merge pull request cli#11316 from cli/babakks/automate-spam-issue-detection
Automate spam issue detection
2 parents 42a8e02 + aa955e1 commit b2348f8

7 files changed

Lines changed: 5348 additions & 0 deletions

File tree

‎.github/workflows/detect-spam.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: Spam Issue Detection
2+
on:
3+
issues:
4+
types: [opened]
5+
6+
permissions:
7+
contents: none
8+
issues: write
9+
models: read
10+
11+
jobs:
12+
issue-spam:
13+
runs-on: ubuntu-latest
14+
environment: cli-automation
15+
steps:
16+
- name: Checkout repository
17+
uses: actions/checkout@v4
18+
- name: Run spam detection
19+
env:
20+
GH_TOKEN: ${{ secrets.AUTOMATION_TOKEN }}
21+
ISSUE_URL: ${{ github.event.issue.html_url }}
22+
run: |
23+
./.github/workflows/scripts/spam-detection/process-issue.sh "$ISSUE_URL"
24+
if [[ $? -ne 0 ]]; then
25+
echo "error processing issue"
26+
exit 1
27+
fi

‎.github/workflows/scripts/spam-detection/check-issue-prompts.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
name: Detect spam
2+
model: openai/gpt-4o-mini
3+
messages:
4+
- role: system
5+
content: "" # Since it's not a fix value, it should be generated and replaced at runtime
6+
- role: user
7+
content: "" # This will be replaced at runtime

‎.github/workflows/scripts/spam-detection/check-issue.sh‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/bin/bash
2+
3+
# Check if an issue is spam or not and output "PASS" (not spam) or "FAIL" (spam).
4+
#
5+
# Regardless of the spam detection result, the script always exits with a zero
6+
# exit code, unless there's a runtime error.
7+
#
8+
# This script must be run from the root directory of the repository.
9+
10+
set -euo pipefail
11+
12+
# Determine absolute path to script directory based on where it is called from.
13+
# This allows the script to be run from any directory.
14+
SPAM_DIR="$(dirname "$(realpath "$0")")"
15+
16+
# Retrieve and prepare information about issue for detection
17+
_issue_url="$1"
18+
if [[ -z "$_issue_url" ]]; then
19+
echo "error: issue URL is empty" >&2
20+
exit 1
21+
fi
22+
23+
_user_prompt_template='
24+
<TITLE>
25+
{{ .title }}
26+
</TITLE>
27+
28+
<BODY>
29+
{{ .body }}
30+
</BODY>
31+
'
32+
33+
_user_prompt="$(gh issue view --json title,body --template "$_user_prompt_template" "$_issue_url")"
34+
35+
# Generate dynamic prompts for inference
36+
_system_prompt="$($SPAM_DIR/generate-sys-prompt.sh)"
37+
_final_prompt="$(_system="$_system_prompt" _user="$_user_prompt" yq eval ".messages[0].content = strenv(_system) | .messages[1].content = strenv(_user)" "$SPAM_DIR/check-issue-prompts.yml")"
38+
39+
gh extension install github/gh-models 2>/dev/null
40+
41+
_result="$(gh models run --file <(echo "$_final_prompt") | cat)"
42+
43+
if [[ "$_result" != "PASS" && "$_result" != "FAIL" ]]; then
44+
echo "error: expected PASS or FAIL but got an unexpected result: $_result" >&2
45+
exit 1
46+
fi
47+
48+
echo "$_result"

0 commit comments

Comments
 (0)