Make individual prompts configurable

Author: sgoedecke

.github/workflows/example-configurable-prompts.yml

@@ -0,0 +1,71 @@
+# Example workflow demonstrating configurable built-in prompts
+name: 'AI Spam Guard with Custom Configuration'
+on:
+  issues:
+    types: [opened]
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+
+jobs:
+  # Basic usage with all prompts enabled (default behavior)
+  full-protection:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+      models: read
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run AI Spam Guard with All Prompts
+        uses: github/ai-spam-guard
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          spam-label: 'spam'
+          ai-label: 'ai-generated'
+          minimize-detected-comments: true
+
+  # Selective prompt configuration example
+  custom-protection:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+      models: read
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run AI Spam Guard with Selective Prompts
+        uses: github/ai-spam-guard
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          spam-label: 'potential-spam'
+          ai-label: 'ai-generated'
+          minimize-detected-comments: false
+          # Only enable general spam and link spam detection
+          enable-spam-detection: true
+          enable-link-spam-detection: true
+          enable-ai-detection: false # Disable AI detection for this workflow
+
+  # Link spam only (for repositories that mainly deal with link spam)
+  link-spam-only:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+      models: read
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run AI Spam Guard - Link Spam Only
+        uses: github/ai-spam-guard
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          spam-label: 'link-spam'
+          minimize-detected-comments: true
+          # Only enable link spam detection
+          enable-spam-detection: false
+          enable-link-spam-detection: true
+          enable-ai-detection: false

README.md

@@ -36,6 +36,10 @@ jobs:
           spam-label: 'spam'
           ai-label: 'ai-generated'
           minimize-detected-comments: true
+          # Built-in prompt configuration (all enabled by default)
+          enable-spam-detection: true
+          enable-link-spam-detection: true
+          enable-ai-detection: true
           # custom-prompt-path: '.github/prompts/my-custom.prompt.yml'  # Optional
 ```
 
@@ -48,6 +52,9 @@ jobs:
 | `ai-label`                   | Label to add when AI-generated content is detected                           | `ai-generated`        | No       |
 | `minimize-detected-comments` | Whether to minimize comments detected as spam                                | `true`                | No       |
 | `custom-prompt-path`         | Path to a custom YAML prompt file in your repository (relative to repo root) | (none)                | No       |
+| `enable-spam-detection`      | Enable built-in spam detection prompt                                        | `true`                | No       |
+| `enable-link-spam-detection` | Enable built-in link spam detection prompt                                   | `true`                | No       |
+| `enable-ai-detection`        | Enable built-in AI-generated content detection prompt                        | `true`                | No       |
 
 ### Inference
 
@@ -60,13 +67,27 @@ into rate limiting issues you can choose to
 
 ## Detection Prompts
 
-The action uses built-in YAML prompts located in the `prompts/` directory:
+The action uses built-in YAML prompts located in the `prompts/` directory. Each
+prompt can be individually enabled or disabled using the configuration options:
 
 - **`spam-detection.prompt.yml`**: Detects promotional content, scams, and
-  irrelevant posts
+  irrelevant posts (controlled by `enable-spam-detection`)
 - **`ai-detection.prompt.yml`**: Identifies AI-generated content patterns
-- **`bot-detection.prompt.yml`**: Identifies automated bot behavior
+  (controlled by `enable-ai-detection`)
 - **`link-spam-detection.prompt.yml`**: Focuses on suspicious links and URLs
+  (controlled by `enable-link-spam-detection`)
+
+All prompts are enabled by default. You can selectively disable them based on
+your repository's moderation needs:
+
+```yaml
+- uses: github/ai-spam-guard
+  with:
+    token: ${{ secrets.GITHUB_TOKEN }}
+    enable-spam-detection: true # Enable general spam detection
+    enable-link-spam-detection: false # Disable link spam detection
+    enable-ai-detection: true # Enable AI content detection
+```
 
 You can iterate on or tweak these prompts via the
 [Models tab](https://github.com/github/ai-spam-guard/models) on this repository.
@@ -85,62 +106,7 @@ You can also provide your own custom prompt file in your repository using the
     custom-prompt-path: '.github/prompts/my-custom-spam-detection.prompt.yml'
 ```
 
-Custom prompt files should follow the same YAML format as the built-in prompts:
-
-```yaml
-messages:
-  - role: system
-    content: |
-      You are a content moderation system. Analyze the provided content
-      and determine if it violates repository guidelines.
-
-      Consider these indicators:
-      - Off-topic discussions
-      - Inappropriate language
-      - Spam or promotional content
-
-      Provide your analysis in the specified JSON format.
-  - role: user
-    content: |
-      Analyze this content:
-
-      {{stdin}}
-model: gpt-4o
-responseFormat: json_schema
-jsonSchema: |-
-  {
-    "name": "spam_detection_result",
-    "strict": true,
-    "schema": {
-      "type": "object",
-      "properties": {
-        "reasoning": {
-          "type": "string",
-          "description": "Detailed explanation of the analysis"
-        },
-        "is_spam": {
-          "type": "boolean",
-          "description": "Whether the content violates guidelines"
-        }
-      },
-      "additionalProperties": false,
-      "required": ["reasoning", "is_spam"]
-    }
-  }
-```
-
-**Notes:**
-
-- Custom prompts are treated as spam detection prompts by default (unless the
-  filename contains "ai-detection")
-- Custom prompts are evaluated **before** the built-in prompts
-- The `{{stdin}}` placeholder will be replaced with the actual content to
-  analyze
-- Custom prompts must return a JSON response with at least `reasoning` and
-  `is_spam` fields
-
-### Example Custom Prompt
-
+Custom prompt files should follow the same YAML format as the built-in prompts.
 An example custom prompt file is included at
 `.github/prompts/example-custom.prompt.yml` that demonstrates the proper format
 and shows how to create repository-specific spam detection rules.

action.yml

@@ -31,6 +31,18 @@ inputs:
       'Path to a custom YAML prompt file in your repository (relative to
       repository root)'
     required: false
+  enable-spam-detection:
+    description: 'Enable built-in spam detection prompt'
+    required: false
+    default: 'true'
+  enable-link-spam-detection:
+    description: 'Enable built-in link spam detection prompt'
+    required: false
+    default: 'true'
+  enable-ai-detection:
+    description: 'Enable built-in AI-generated content detection prompt'
+    required: false
+    default: 'true'
 
 runs:
   using: 'node16'

badges/coverage.svg

@@ -101,7 +101,8 @@ testData:
       multi agents haven’t been so much effective. Then I went back to a single
       agent creating code with baby steps. How it’s going for you?
     expected: 'false'
-  - stdin: this beast with claude 4 opus just gonna eat through your bank account :)
+  - stdin:
+      this beast with claude 4 opus just gonna eat through your bank account :)
     expected: 'false'
 evaluators:
   - name: is-ai

dist/index.js

@@ -18,6 +18,13 @@ async function run(): Promise<void> {
     const minimizeComments = core.getBooleanInput('minimize-detected-comments')
     const customPromptPath = core.getInput('custom-prompt-path')
 
+    // Built-in prompt configuration
+    const enableSpamDetection = core.getBooleanInput('enable-spam-detection')
+    const enableLinkSpamDetection = core.getBooleanInput(
+      'enable-link-spam-detection'
+    )
+    const enableAiDetection = core.getBooleanInput('enable-ai-detection')
+
     const openai = new OpenAI({
       apiKey: token,
       baseURL: 'https://models.github.ai/inference'
@@ -44,7 +51,12 @@ async function run(): Promise<void> {
       openai,
       promptsDir,
       content,
-      customPromptPath
+      customPromptPath,
+      {
+        enableSpamDetection,
+        enableLinkSpamDetection,
+        enableAiDetection
+      }
     )
 
     if (!flags.spam && !flags.ai) {

dist/index.js.map

@@ -15,6 +15,12 @@ interface PromptConfig {
   jsonSchema?: string
 }
 
+interface BuiltInPromptConfig {
+  enableSpamDetection: boolean
+  enableLinkSpamDetection: boolean
+  enableAiDetection: boolean
+}
+
 // Response interfaces for different detection types
 interface SpamDetectionResult {
   reasoning: string
@@ -141,13 +147,30 @@ export async function evaluateContent(
   openai: OpenAI,
   promptsDir: string,
   content: string,
-  customPromptPath?: string
+  customPromptPath?: string,
+  builtInConfig?: BuiltInPromptConfig
 ): Promise<{ spam: boolean; ai: boolean }> {
   const files = await getPromptFiles(promptsDir)
   const flags = { spam: false, ai: false }
 
+  // Filter built-in prompts based on configuration
+  const filteredFiles = files.filter((file) => {
+    const filename = basename(file).toLowerCase()
+
+    if (filename.includes('ai-detection')) {
+      return builtInConfig?.enableAiDetection !== false
+    } else if (filename.includes('spam-detection')) {
+      return builtInConfig?.enableSpamDetection !== false
+    } else if (filename.includes('link-spam')) {
+      return builtInConfig?.enableLinkSpamDetection !== false
+    }
+
+    // Include unknown prompt types by default
+    return true
+  })
+
   // Add custom prompt file if provided
-  const allFiles = [...files]
+  const allFiles = [...filteredFiles]
   if (customPromptPath && customPromptPath.trim()) {
     try {
       // Resolve custom prompt path relative to workspace root