Skip to content

Commit be2b5a1

Browse files
data-douserdata-douser
committed
actions: experimental query for CWE-1426 AI (output)
Add detection for improper validation of AI-generated output (CWE-1426) in GitHub Actions workflows where AI action output flows unsanitized to code execution sinks. New query: - ImproperValidationOfAiOutputCritical.ql: Detects AI-generated output flowing to run steps or subsequent AI prompts in privileged contexts (severity 9.0) New library: - ImproperValidationOfAiOutputQuery.qll: Taint tracking from AI action output references to code execution and AI inference sinks MaD model (ai_inference_actions.model.yml): - 15 AI actions identified as AI inference sources whose outputs should be treated as untrusted
1 parent 16683ae commit be2b5a1

File tree

12 files changed

+368
-0
lines changed

12 files changed

+368
-0
lines changed

‎actions/ql/lib/codeql/actions/security/ImproperValidationOfAiOutputQuery.qll‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
/**
2+
* Provides classes and predicates for detecting improper validation of
3+
* generative AI output in GitHub Actions workflows (CWE-1426).
4+
*
5+
* This library identifies cases where AI-generated output flows unsanitized
6+
* into code execution sinks (LOTP gadgets) or subsequent AI prompts.
7+
*/
8+
9+
private import actions
10+
private import codeql.actions.TaintTracking
11+
private import codeql.actions.dataflow.ExternalFlow
12+
import codeql.actions.dataflow.FlowSources
13+
import codeql.actions.DataFlow
14+
import codeql.actions.security.ControlChecks
15+
16+
/**
17+
* A source representing AI-generated output from AI inference actions.
18+
* This models CWE-1426 where AI output is used without proper validation,
19+
* potentially allowing an attacker to chain prompt injection into code execution.
20+
*/
21+
class AiInferenceOutputSource extends DataFlow::Node {
22+
UsesStep aiStep;
23+
24+
AiInferenceOutputSource() {
25+
exists(StepsExpression stepRef, string action |
26+
this.asExpr() = stepRef and
27+
stepRef.getStepId() = aiStep.getId() and
28+
actionsSinkModel(action, _, _, "ai-inference", _) and
29+
aiStep.getCallee() = action
30+
)
31+
}
32+
33+
/** Gets the AI inference step that produces this output. */
34+
UsesStep getAiStep() { result = aiStep }
35+
}
36+
37+
/**
38+
* A sink for improper validation of AI output (CWE-1426).
39+
* AI output flowing unsanitized to code execution (LOTP gadgets),
40+
* subsequent AI prompts, or environment manipulation.
41+
*/
42+
class ImproperAiOutputSink extends DataFlow::Node {
43+
ImproperAiOutputSink() {
44+
// Code injection sinks (run steps) — LOTP gadgets
45+
exists(Run e | e.getAnScriptExpr() = this.asExpr())
46+
or
47+
// MaD-defined code injection sinks
48+
madSink(this, "code-injection")
49+
or
50+
// AI inference sinks (AI output flowing to another AI prompt = chained injection)
51+
madSink(this, "ai-inference")
52+
}
53+
}
54+
55+
/**
56+
* Gets the relevant event for sinks in a privileged context.
57+
*/
58+
Event getRelevantEventForAiOutputSink(DataFlow::Node sink) {
59+
inPrivilegedContext(sink.asExpr(), result) and
60+
not exists(ControlCheck check | check.protects(sink.asExpr(), result, "code-injection"))
61+
}
62+
63+
/**
64+
* Holds when a critical-severity AI output validation issue exists.
65+
*/
66+
predicate criticalAiOutputInjection(
67+
ImproperAiOutputFlow::PathNode source, ImproperAiOutputFlow::PathNode sink, Event event
68+
) {
69+
ImproperAiOutputFlow::flowPath(source, sink) and
70+
event = getRelevantEventForAiOutputSink(sink.getNode())
71+
}
72+
73+
/**
74+
* A taint-tracking configuration for AI-generated output
75+
* that flows unsanitized to code execution or subsequent AI prompts.
76+
*/
77+
private module ImproperAiOutputConfig implements DataFlow::ConfigSig {
78+
predicate isSource(DataFlow::Node source) { source instanceof AiInferenceOutputSource }
79+
80+
predicate isSink(DataFlow::Node sink) { sink instanceof ImproperAiOutputSink }
81+
82+
predicate observeDiffInformedIncrementalMode() { any() }
83+
84+
Location getASelectedSinkLocation(DataFlow::Node sink) {
85+
result = sink.getLocation()
86+
or
87+
result = getRelevantEventForAiOutputSink(sink).getLocation()
88+
}
89+
}
90+
91+
/** Tracks flow of AI-generated output to code execution sinks. */
92+
module ImproperAiOutputFlow = TaintTracking::Global<ImproperAiOutputConfig>;

‎actions/ql/lib/ext/manual/ai_inference_actions.model.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
extensions:
2+
- addsTo:
3+
pack: codeql/actions-all
4+
extensible: actionsSinkModel
5+
# AI inference actions whose output should be treated as untrusted.
6+
# Used by CWE-1426 (ImproperValidationOfAiOutput) to identify AI action steps
7+
# whose outputs may flow unsanitized to code execution sinks (LOTP gadgets).
8+
# source: https://boostsecurityio.github.io/lotp/
9+
# source: https://github.com/marketplace?type=actions&category=ai-assisted
10+
data:
11+
# === GitHub official ===
12+
- ["actions/ai-inference", "*", "input.prompt", "ai-inference", "manual"]
13+
- ["github/ai-moderator", "*", "input.prompt", "ai-inference", "manual"]
14+
# === Anthropic ===
15+
- ["anthropics/claude-code-action", "*", "input.prompt", "ai-inference", "manual"]
16+
# === Google ===
17+
- ["google/gemini-code-assist-action", "*", "input.prompt", "ai-inference", "manual"]
18+
- ["google-gemini/code-assist-action", "*", "input.prompt", "ai-inference", "manual"]
19+
# === OpenAI ===
20+
- ["openai/chat-completion-action", "*", "input.prompt", "ai-inference", "manual"]
21+
# === Community AI review/inference actions ===
22+
- ["coderabbitai/ai-pr-reviewer", "*", "input.prompt", "ai-inference", "manual"]
23+
- ["CodiumAI/pr-agent", "*", "input.prompt", "ai-inference", "manual"]
24+
- ["platisd/openai-pr-description", "*", "input.prompt", "ai-inference", "manual"]
25+
- ["austenstone/openai-completion-action", "*", "input.prompt", "ai-inference", "manual"]
26+
- ["github/copilot-text-inference", "*", "input.prompt", "ai-inference", "manual"]
27+
- ["huggingface/inference-action", "*", "input.prompt", "ai-inference", "manual"]
28+
- ["replicate/action", "*", "input.prompt", "ai-inference", "manual"]
29+
# === Google (GitHub Actions org) ===
30+
- ["google-github-actions/run-gemini-cli", "*", "input.prompt", "ai-inference", "manual"]
31+
# === Warp ===
32+
- ["warpdotdev/oz-agent-action", "*", "input.prompt", "ai-inference", "manual"]

‎actions/ql/src/experimental/Security/CWE-1426/ImproperValidationOfAiOutputCritical.md‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
## Overview
2+
3+
Using AI-generated output without validation in GitHub Actions workflows enables **chained injection attacks** where an attacker's prompt injection in one step produces malicious AI output that executes as code in a subsequent step. When AI output flows unsanitized into shell commands, build scripts, package installations, or subsequent AI prompts, an attacker who controls the AI's input effectively controls the code that runs in your CI/CD pipeline.
4+
5+
AI output should always be treated as untrusted data. This is especially dangerous because the malicious payload is generated dynamically by the AI and may bypass traditional static analysis or code review.
6+
7+
## Recommendation
8+
9+
Treat all AI-generated output as untrusted. Before using AI output in any executable context:
10+
11+
- **Validate the format** — check that the output matches an expected schema or pattern before use.
12+
- **Never interpolate AI output directly into `run:` steps** — use environment variables and validate before execution.
13+
- **Limit AI action permissions** — restrict `GITHUB_TOKEN` scope and avoid passing secrets to workflows that consume AI output.
14+
- **Use structured output formats** (e.g. JSON with a defined schema) to constrain AI responses and make validation easier.
15+
- **Avoid chaining AI calls** without validating intermediate output. Each AI step's output is a potential injection vector for the next.
16+
17+
## Example
18+
19+
### Incorrect Usage
20+
21+
The following example executes AI output directly as a shell command. An attacker who controls the AI's input (via the issue body) can cause the AI to output arbitrary shell commands:
22+
23+
```yaml
24+
on:
25+
issues:
26+
types: [opened]
27+
28+
jobs:
29+
ai-task:
30+
runs-on: ubuntu-latest
31+
steps:
32+
- name: AI inference
33+
id: ai
34+
uses: actions/ai-inference@v1
35+
with:
36+
prompt: |
37+
Suggest a fix for: ${{ github.event.issue.body }}
38+
39+
- name: Apply fix
40+
run: |
41+
${{ steps.ai.outputs.response }}
42+
```
43+
44+
### Correct Usage
45+
46+
The following example validates the AI output format before taking any action:
47+
48+
```yaml
49+
- name: Validate and apply
50+
run: |
51+
RESPONSE="${AI_RESPONSE}"
52+
# Only accept responses that match a safe pattern
53+
if echo "$RESPONSE" | grep -qE '^(fix|patch|update):'; then
54+
echo "Valid response format, proceeding"
55+
else
56+
echo "::warning::Unexpected AI output format, skipping execution"
57+
exit 0
58+
fi
59+
env:
60+
AI_RESPONSE: ${{ steps.ai.outputs.response }}
61+
```
62+
63+
## References
64+
65+
- Common Weakness Enumeration: [CWE-1426](https://cwe.mitre.org/data/definitions/1426.html).
66+
- [OWASP LLM02: Insecure Output Handling](https://genai.owasp.org/llmrisk/llm02-insecure-output-handling/).
67+
- GitHub Docs: [Security hardening for GitHub Actions](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions).

‎actions/ql/src/experimental/Security/CWE-1426/ImproperValidationOfAiOutputCritical.ql‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/**
2+
* @name Improper validation of AI-generated output
3+
* @description AI-generated output flowing unsanitized to code execution or
4+
* subsequent AI prompts may allow chained prompt injection attacks.
5+
* @kind path-problem
6+
* @problem.severity error
7+
* @security-severity 9.0
8+
* @precision high
9+
* @id actions/improper-validation-of-ai-output/critical
10+
* @tags actions
11+
* security
12+
* experimental
13+
* external/cwe/cwe-1426
14+
*/
15+
16+
import actions
17+
import codeql.actions.security.ImproperValidationOfAiOutputQuery
18+
import ImproperAiOutputFlow::PathGraph
19+
20+
from ImproperAiOutputFlow::PathNode source, ImproperAiOutputFlow::PathNode sink, Event event
21+
where criticalAiOutputInjection(source, sink, event)
22+
select sink.getNode(), source, sink,
23+
"AI-generated output flows unsanitized to $@, which may allow chained injection ($@).", sink,
24+
sink.getNode().asExpr().(Expression).getRawExpression(), event, event.getName()

‎actions/ql/test/query-tests/Security/CWE-1426/.github/workflows/safe1.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
name: Safe AI Output Usage
2+
on:
3+
push:
4+
branches: [main]
5+
6+
jobs:
7+
analyze:
8+
runs-on: ubuntu-latest
9+
steps:
10+
- name: AI inference
11+
id: ai
12+
uses: actions/ai-inference@v1
13+
with:
14+
prompt: |
15+
Analyze this repository.
16+
17+
- name: Display result only
18+
run: |
19+
echo "AI said something"

‎actions/ql/test/query-tests/Security/CWE-1426/.github/workflows/safe2.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: Safe AI Output to Comment Only
2+
on:
3+
issues:
4+
types: [opened]
5+
6+
jobs:
7+
respond:
8+
runs-on: ubuntu-latest
9+
permissions:
10+
issues: write
11+
steps:
12+
- name: AI analysis
13+
id: ai
14+
uses: actions/ai-inference@v1
15+
with:
16+
prompt: |
17+
Summarize this issue.
18+
19+
- name: Post comment with AI response
20+
uses: actions/github-script@v7
21+
with:
22+
script: |
23+
github.rest.issues.createComment({
24+
owner: context.repo.owner,
25+
repo: context.repo.repo,
26+
issue_number: context.issue.number,
27+
body: 'AI Summary posted'
28+
})

‎actions/ql/test/query-tests/Security/CWE-1426/.github/workflows/vulnerable1.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: AI Output to Shell
2+
on:
3+
issues:
4+
types: [opened]
5+
6+
jobs:
7+
ai-task:
8+
runs-on: ubuntu-latest
9+
permissions:
10+
issues: write
11+
models: read
12+
steps:
13+
- name: AI inference
14+
id: ai
15+
uses: actions/ai-inference@v1
16+
with:
17+
prompt: |
18+
Suggest a fix for: ${{ github.event.issue.body }}
19+
20+
- name: Apply fix unsanitized
21+
run: |
22+
${{ steps.ai.outputs.response }}

‎actions/ql/test/query-tests/Security/CWE-1426/.github/workflows/vulnerable2.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: AI Output Chain
2+
on:
3+
issues:
4+
types: [opened]
5+
6+
jobs:
7+
ai-task:
8+
runs-on: ubuntu-latest
9+
permissions:
10+
issues: write
11+
models: read
12+
steps:
13+
- name: First AI inference
14+
id: ai1
15+
uses: actions/ai-inference@v1
16+
with:
17+
prompt: |
18+
Summarize: ${{ github.event.issue.body }}
19+
20+
- name: Second AI inference using first AI output
21+
id: ai2
22+
uses: actions/ai-inference@v1
23+
with:
24+
prompt: |
25+
Improve this summary:
26+
${{ steps.ai1.outputs.response }}

‎actions/ql/test/query-tests/Security/CWE-1426/.github/workflows/vulnerable3.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: Claude Output to Shell
2+
on:
3+
issues:
4+
types: [opened]
5+
6+
jobs:
7+
fix:
8+
runs-on: ubuntu-latest
9+
permissions:
10+
contents: write
11+
issues: write
12+
steps:
13+
- name: Claude analysis
14+
id: claude
15+
uses: anthropics/claude-code-action@v1
16+
with:
17+
prompt: |
18+
Suggest a shell command to fix this issue:
19+
${{ github.event.issue.title }}
20+
21+
- name: Execute AI suggestion
22+
run: |
23+
${{ steps.claude.outputs.response }}

‎actions/ql/test/query-tests/Security/CWE-1426/.github/workflows/vulnerable4.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: Gemini Output to Shell
2+
on:
3+
pull_request_review:
4+
types: [submitted]
5+
6+
jobs:
7+
apply-fix:
8+
runs-on: ubuntu-latest
9+
permissions:
10+
contents: write
11+
pull-requests: write
12+
steps:
13+
- name: Gemini review
14+
id: gemini
15+
uses: google-github-actions/run-gemini-cli@v1
16+
with:
17+
prompt: |
18+
Suggest a patch for this PR.
19+
20+
- name: Apply Gemini suggestion
21+
run: |
22+
echo "${{ steps.gemini.outputs.response }}" | patch -p1

0 commit comments

Comments
 (0)