Skip to content

feat: add rule for client-side encryption of sensitive fields #133

feat: add rule for client-side encryption of sensitive fields

feat: add rule for client-side encryption of sensitive fields #133

# =============================================================================
# Aggregate Batch Results
# =============================================================================
# Collects test results from multiple child PRs in a batch, computes aggregate
# statistics (mean, stddev, min, max), and creates a summary PR with the
# analysis. Closes the child PRs after collecting results.
#
# Triggers:
# - issue_comment: Post "/aggregate 51,52,53,54,55" on the parent batch issue
# - workflow_dispatch: Manual trigger from the Actions tab
#
# What it does:
# 1. Downloads test-report.json artifacts from each child PR
# 2. Runs aggregate.py to compute statistics
# 3. Creates a summary branch and PR with BATCH-RESULTS.md
# 4. Closes child PRs
# 5. Posts aggregate summary on the parent batch issue
# =============================================================================
name: Aggregate Batch Results
on:
issue_comment:
types: [created]
workflow_dispatch:
inputs:
batch_issue:
description: 'Parent batch issue number'
required: true
type: number
child_issues:
description: 'Comma-separated child issue numbers (e.g., 61,62,63,64,65)'
required: true
type: string
jobs:
aggregate:
name: Aggregate batch results
runs-on: ubuntu-latest
permissions:
issues: write
pull-requests: write
contents: write
# For issue_comment: only run on /aggregate commands on batch-test issues
if: >
github.event_name == 'workflow_dispatch' ||
(github.event_name == 'issue_comment'
&& startsWith(github.event.comment.body, '/aggregate')
&& contains(github.event.issue.labels.*.name, 'batch-test'))
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Verify commenter permission
if: github.event_name == 'issue_comment'
uses: actions/github-script@v7
with:
script: |
const commenter = context.payload.comment.user.login;
const { data: perm } = await github.rest.repos.getCollaboratorPermissionLevel({
owner: context.repo.owner,
repo: context.repo.repo,
username: commenter,
});
if (perm.permission !== 'admin' && perm.permission !== 'write') {
core.setFailed(`User ${commenter} does not have write access`);
}
- name: Parse trigger inputs
id: inputs
env:
GH_TOKEN: ${{ github.token }}
COMMENT_BODY: ${{ github.event.comment.body }}
run: |
if [ "${{ github.event_name }}" = "issue_comment" ]; then
# Parse from comment: /aggregate 61,62,63,64,65
# COMMENT_BODY is passed via env to avoid script injection (CWE-94)
CHILD_ISSUES=$(echo "$COMMENT_BODY" | sed 's|/aggregate[[:space:]]*||' | tr -d '[:space:]')
# Validate that CHILD_ISSUES contains only digits and commas
if ! echo "$CHILD_ISSUES" | grep -qE '^[0-9]+(,[0-9]+)*$'; then
echo "ERROR: Invalid child issues format. Expected comma-separated numbers."
exit 1
fi
BATCH_ISSUE=${{ github.event.issue.number }}
echo "Triggered via issue comment on #${BATCH_ISSUE}"
else
CHILD_ISSUES="${{ inputs.child_issues }}"
BATCH_ISSUE=${{ inputs.batch_issue }}
echo "Triggered via workflow_dispatch"
fi
echo "batch_issue=${BATCH_ISSUE}" >> "$GITHUB_OUTPUT"
echo "child_issues=${CHILD_ISSUES}" >> "$GITHUB_OUTPUT"
echo "Batch issue: #${BATCH_ISSUE}, Child issues: ${CHILD_ISSUES}"
- name: Resolve child issues to PRs
id: resolve
env:
GH_TOKEN: ${{ github.token }}
run: |
IFS=',' read -ra ISSUE_LIST <<< "${{ steps.inputs.outputs.child_issues }}"
PR_LIST=()
for ISSUE_NUM in "${ISSUE_LIST[@]}"; do
ISSUE_NUM=$(echo "$ISSUE_NUM" | tr -d '[:space:]')
echo "Resolving issue #${ISSUE_NUM} to PR..."
# Primary: use timeline API to find cross-referenced PRs
# This is the most reliable method — GitHub tracks when Copilot
# creates a PR that references an issue (e.g., "Fixes #82")
PR_NUM=$(gh api "repos/${{ github.repository }}/issues/${ISSUE_NUM}/timeline" \
--jq '[.[] | select(.event == "cross-referenced" and .source.issue.pull_request != null)] | .[0].source.issue.number' \
2>/dev/null || echo "")
# Fallback: search for PRs mentioning this issue number in the body
if [ -z "$PR_NUM" ] || [ "$PR_NUM" = "null" ]; then
echo " Timeline API found no PR, trying body search..."
PR_NUM=$(gh pr list --state all --search "in:body \"Fixes #${ISSUE_NUM}\"" \
--json number --jq '.[0].number' 2>/dev/null || echo "")
fi
# Fallback: search PR titles for the issue's iteration name
if [ -z "$PR_NUM" ] || [ "$PR_NUM" = "null" ]; then
echo " Body search found no PR, trying iteration-name title search..."
ISSUE_BODY=$(gh issue view "$ISSUE_NUM" --json body --jq '.body')
ITER_NAME=$(printf '%s' "$ISSUE_BODY" | grep -oE 'iteration-[0-9]{3}-[A-Za-z0-9._-]+' | head -1)
if [ -n "$ITER_NAME" ]; then
PR_NUM=$(gh pr list --state all --search "in:title \"${ITER_NAME}\"" \
--json number --jq '.[0].number' 2>/dev/null || echo "")
else
echo " WARNING: Could not extract iteration name from issue #${ISSUE_NUM}"
fi
fi
# Final fallback: search PR titles for the full child issue title
if [ -z "$PR_NUM" ] || [ "$PR_NUM" = "null" ]; then
echo " Iteration-name search found no PR, trying full issue title..."
ISSUE_TITLE=$(gh issue view "$ISSUE_NUM" --json title --jq '.title')
PR_NUM=$(gh pr list --state all --search "in:title \"${ISSUE_TITLE}\"" \
--json number --jq '.[0].number' 2>/dev/null || echo "")
fi
if [ -n "$PR_NUM" ] && [ "$PR_NUM" != "null" ]; then
echo " Issue #${ISSUE_NUM} → PR #${PR_NUM}"
PR_LIST+=("$PR_NUM")
else
echo " WARNING: No PR found for issue #${ISSUE_NUM}"
fi
done
PR_NUMBERS=$(IFS=','; echo "${PR_LIST[*]}")
echo "pr_numbers=${PR_NUMBERS}" >> "$GITHUB_OUTPUT"
echo "Resolved PRs: ${PR_NUMBERS}"
if [ ${#PR_LIST[@]} -eq 0 ]; then
echo "ERROR: Could not resolve any child issues to PRs."
exit 1
fi
- name: Acknowledge aggregation started
if: github.event_name == 'issue_comment'
env:
GH_TOKEN: ${{ github.token }}
run: |
gh issue comment ${{ steps.inputs.outputs.batch_issue }} \
--body "Aggregation started for child issues: #$(echo '${{ steps.inputs.outputs.child_issues }}' | sed 's/,/, #/g'). Resolving PRs and collecting test results..."
- name: Parse batch issue metadata
id: meta
env:
GH_TOKEN: ${{ github.token }}
run: |
BATCH_ISSUE=${{ steps.inputs.outputs.batch_issue }}
BODY=$(gh issue view "$BATCH_ISSUE" --json body --jq '.body')
# Parse form fields
parse_field() {
local label="$1"
echo "$BODY" | sed -n "/### ${label}/,/### /{/### ${label}/d;/### /d;/^$/d;p;}" | head -1 | tr -d '[:space:]'
}
SCENARIO=$(parse_field "Scenario")
LANGUAGE=$(parse_field "Language")
SKILLS=$(parse_field "Load skills?")
ITERATIONS=$(parse_field "Number of iterations")
echo "scenario=$SCENARIO" >> "$GITHUB_OUTPUT"
echo "language=$LANGUAGE" >> "$GITHUB_OUTPUT"
echo "skills=$SKILLS" >> "$GITHUB_OUTPUT"
echo "iterations=$ITERATIONS" >> "$GITHUB_OUTPUT"
echo "Parsed: scenario=$SCENARIO language=$LANGUAGE skills=$SKILLS iterations=$ITERATIONS"
- name: Download artifacts from child PRs
env:
GH_TOKEN: ${{ github.token }}
run: |
mkdir -p batch-results
IFS=',' read -ra PR_LIST <<< "${{ steps.resolve.outputs.pr_numbers }}"
RUN_INDEX=0
for PR_NUM in "${PR_LIST[@]}"; do
PR_NUM=$(echo "$PR_NUM" | tr -d '[:space:]')
RUN_INDEX=$((RUN_INDEX + 1))
echo ""
echo "=== Processing PR #${PR_NUM} (run ${RUN_INDEX}) ==="
# Get the PR's head branch
HEAD_BRANCH=$(gh pr view "$PR_NUM" --json headRefName --jq '.headRefName')
echo " Head branch: $HEAD_BRANCH"
# Find the workflow run that has test-results artifacts
# We search by branch name and iterate through completed runs
# because pull_request_target runs don't match the PR's head SHA,
# and Copilot agent runs also appear on the same branch without artifacts.
ARTIFACT_ID=""
RUN_ID=""
CANDIDATE_RUNS=$(gh api "repos/${{ github.repository }}/actions/runs" \
--method GET \
-f branch="$HEAD_BRANCH" \
-f status=completed \
--jq '.workflow_runs[].id' 2>/dev/null || echo "")
for CANDIDATE_RUN in $CANDIDATE_RUNS; do
FOUND_ARTIFACT=$(gh api "repos/${{ github.repository }}/actions/runs/${CANDIDATE_RUN}/artifacts" \
--jq '.artifacts[] | select(.name | startswith("test-results-")) | .id' 2>/dev/null || echo "")
if [ -n "$FOUND_ARTIFACT" ]; then
RUN_ID="$CANDIDATE_RUN"
ARTIFACT_ID=$(echo "$FOUND_ARTIFACT" | head -1)
echo " Found test artifact in run $RUN_ID"
break
fi
done
if [ -z "$ARTIFACT_ID" ]; then
echo " WARNING: No test-results artifact found across any runs for PR #${PR_NUM}"
continue
fi
echo " Workflow run ID: $RUN_ID"
echo " Artifact ID: $ARTIFACT_ID"
# Download and extract the artifact
RUN_DIR="batch-results/run-${RUN_INDEX}"
mkdir -p "$RUN_DIR"
gh api "repos/${{ github.repository }}/actions/artifacts/${ARTIFACT_ID}/zip" > "${RUN_DIR}/artifact.zip"
unzip -o "${RUN_DIR}/artifact.zip" -d "$RUN_DIR" 2>/dev/null || true
rm -f "${RUN_DIR}/artifact.zip"
# Verify we got test-report.json
if [ -f "${RUN_DIR}/test-report.json" ]; then
echo " Downloaded test-report.json"
else
echo " WARNING: test-report.json not found in artifact"
ls -la "$RUN_DIR/" || true
fi
done
# Count successful downloads
FOUND=$(find batch-results -name "test-report.json" | wc -l)
echo ""
echo "Downloaded ${FOUND} test-report.json files from ${#PR_LIST[@]} PRs"
if [ "$FOUND" -eq 0 ]; then
echo "ERROR: No test results found. Cannot aggregate."
exit 1
fi
- name: Run aggregation
id: aggregate
env:
SCENARIO: ${{ steps.meta.outputs.scenario }}
LANGUAGE: ${{ steps.meta.outputs.language }}
SKILLS: ${{ steps.meta.outputs.skills }}
BATCH_ISSUE: ${{ steps.inputs.outputs.batch_issue }}
PR_NUMBERS: ${{ steps.resolve.outputs.pr_numbers }}
run: |
# Collect all test-report.json paths
REPORTS=$(find batch-results -name "test-report.json" | sort)
echo "Found reports:"
echo "$REPORTS"
# Run the aggregation script
python testing-v2/harness/aggregate.py \
--reports $REPORTS \
--scenario "$SCENARIO" \
--language "$LANGUAGE" \
--skills "$SKILLS" \
--batch-issue "$BATCH_ISSUE" \
--pr-numbers "$PR_NUMBERS" \
--output-md batch-results/BATCH-RESULTS.md \
--output-json batch-results/batch-results.json
echo "Aggregation complete"
cat batch-results/BATCH-RESULTS.md
- name: Commit batch results to branch
env:
GH_TOKEN: ${{ github.token }}
SCENARIO: ${{ steps.meta.outputs.scenario }}
LANGUAGE: ${{ steps.meta.outputs.language }}
SKILLS: ${{ steps.meta.outputs.skills }}
BATCH_ISSUE: ${{ steps.inputs.outputs.batch_issue }}
run: |
SKILLS_LABEL=$( [ "$SKILLS" = "yes" ] && echo "skills" || echo "control" )
BRANCH="batch-${BATCH_ISSUE}-${SCENARIO}-${SKILLS_LABEL}"
RESULTS_DIR="testing-v2/scenarios/${SCENARIO}/batch-results"
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
# Create branch from the current branch
git checkout -b "$BRANCH"
# Copy batch results to the repo
mkdir -p "$RESULTS_DIR"
cp batch-results/BATCH-RESULTS.md "$RESULTS_DIR/batch-${BATCH_ISSUE}-${LANGUAGE}-${SKILLS_LABEL}.md"
cp batch-results/batch-results.json "$RESULTS_DIR/batch-${BATCH_ISSUE}-${LANGUAGE}-${SKILLS_LABEL}.json"
# Update IMPROVEMENTS-LOG.md
LOG_FILE="testing-v2/IMPROVEMENTS-LOG.md"
ENTRY="| $(date -u +%Y-%m-%d) | ${SCENARIO} | Batch #${BATCH_ISSUE} (${SKILLS_LABEL}, ${LANGUAGE}) | Aggregated $(echo "${{ steps.resolve.outputs.pr_numbers }}" | tr ',' ' ' | wc -w | tr -d ' ') iterations | See batch-results/ |"
if [ -f "$LOG_FILE" ]; then
# Insert after the header row (line with |---|)
sed -i "/^|.*---|/a\\${ENTRY}" "$LOG_FILE"
fi
git add "$RESULTS_DIR/" "$LOG_FILE" "CHANGELOG.md"
git commit -m "Batch #${BATCH_ISSUE}: aggregate results for ${SCENARIO} (${SKILLS_LABEL}, ${LANGUAGE}) [skip ci]"
git push origin "$BRANCH"
echo "branch=${BRANCH}" >> "$GITHUB_OUTPUT"
id: commit_results
- name: Create summary PR
env:
GH_TOKEN: ${{ github.token }}
SCENARIO: ${{ steps.meta.outputs.scenario }}
LANGUAGE: ${{ steps.meta.outputs.language }}
SKILLS: ${{ steps.meta.outputs.skills }}
BATCH_ISSUE: ${{ steps.inputs.outputs.batch_issue }}
BRANCH: ${{ steps.commit_results.outputs.branch }}
run: |
# Ensure labels exist
gh label create "batch-aggregate" --description "Batch aggregate summary PR" --color "0e8a16" 2>/dev/null || true
gh label create "testing" --description "Testing framework" --color "1d76db" 2>/dev/null || true
SKILLS_TEXT=$( [ "$SKILLS" = "yes" ] && echo "skills loaded" || echo "control (no skills)" )
PR_TITLE="batch: ${SCENARIO} aggregate results — ${SKILLS_TEXT} (${LANGUAGE})"
if [ "$SKILLS" = "yes" ]; then
NEXT_STEPS="Review the **Consistent Failures** section. These tests failed in every iteration and indicate real skill gaps. Use the deep evaluation prompt (posted as a comment) to analyze and create rules."
else
NEXT_STEPS="This is a **control run** (no skills loaded). Review the consistent failures to identify which existing rules WOULD HAVE helped. Compare with the corresponding skills batch to measure skill effectiveness."
fi
PR_BODY=$(cat <<EOF
## Batch Test Aggregate Results
**Parent issue**: #${BATCH_ISSUE}
**Scenario**: ${SCENARIO}
**Language**: ${LANGUAGE}
**Skills**: ${SKILLS_TEXT}
**Child PRs**: $(echo "${{ steps.resolve.outputs.pr_numbers }}" | sed 's/,/, #/g; s/^/#/')
---
$(cat batch-results/BATCH-RESULTS.md)
---
### Next Steps
${NEXT_STEPS}
EOF
)
# Create PR and capture output separately from errors
PR_URL=$(gh pr create \
--title "$PR_TITLE" \
--body "$PR_BODY" \
--label "testing" \
--label "batch-aggregate" \
--head "$BRANCH") || true
if [ -n "$PR_URL" ]; then
SUMMARY_PR=$(echo "$PR_URL" | grep -oP '\d+$')
echo "Created summary PR #${SUMMARY_PR}: ${PR_URL}"
else
echo "WARNING: Failed to create summary PR"
SUMMARY_PR=""
fi
echo "summary_pr=${SUMMARY_PR}" >> "$GITHUB_OUTPUT"
id: create_summary
- name: Close child PRs
env:
GH_TOKEN: ${{ github.token }}
run: |
IFS=',' read -ra PR_LIST <<< "${{ steps.resolve.outputs.pr_numbers }}"
for PR_NUM in "${PR_LIST[@]}"; do
PR_NUM=$(echo "$PR_NUM" | tr -d '[:space:]')
echo "Closing child PR #${PR_NUM}..."
gh pr close "$PR_NUM" \
--comment "Closed as part of batch aggregation. Results aggregated in batch #${{ steps.inputs.outputs.batch_issue }}." \
--delete-branch \
2>/dev/null || echo " WARNING: Could not close PR #${PR_NUM}"
done
- name: Post aggregate summary on parent issue
env:
GH_TOKEN: ${{ github.token }}
run: |
BATCH_ISSUE=${{ steps.inputs.outputs.batch_issue }}
RESULTS=$(cat batch-results/BATCH-RESULTS.md)
SUMMARY_PR="${{ steps.create_summary.outputs.summary_pr }}"
if [ -n "$SUMMARY_PR" ]; then
PR_NOTE="*Summary PR #${SUMMARY_PR} created for deep evaluation. See linked PR for full details.*"
else
PR_NOTE="*WARNING: Summary PR creation failed. Results are still available in the artifacts.*"
fi
COMMENT="## Batch Aggregation Complete
All child PRs have been processed and closed. Results:
---
${RESULTS}
---
${PR_NOTE}"
gh issue comment "$BATCH_ISSUE" --body "$COMMENT"
echo "Posted aggregate summary on batch issue #${BATCH_ISSUE}"
- name: Post deep evaluation prompt on summary PR
if: steps.create_summary.outputs.summary_pr
env:
GH_TOKEN: ${{ github.token }}
SCENARIO: ${{ steps.meta.outputs.scenario }}
LANGUAGE: ${{ steps.meta.outputs.language }}
SKILLS: ${{ steps.meta.outputs.skills }}
BATCH_ISSUE: ${{ steps.inputs.outputs.batch_issue }}
run: |
SUMMARY_PR=${{ steps.create_summary.outputs.summary_pr }}
SKILLS_LABEL=$( [ "$SKILLS" = "yes" ] && echo "skills" || echo "control" )
RESULTS_FILE="testing-v2/scenarios/${SCENARIO}/batch-results/batch-${BATCH_ISSUE}-${LANGUAGE}-${SKILLS_LABEL}.md"
# Build the evaluation comment as a file to avoid escaping issues
cat > /tmp/eval-comment.md <<EVALEOF
<!-- copilot-batch-eval -->
## 📋 Deep Evaluation Ready$( [ "$SKILLS" = "yes" ] && echo "" || echo " (CONTROL RUN)" )
Batch aggregation is complete. A deep evaluation is needed to analyze the consistently-failing tests.
---
### 🔧 Action Required
> **Copy the prompt below and post it as a new comment** to trigger Copilot to perform the deep evaluation.
> (Bot-posted \`@copilot\` mentions do not trigger the agent — it must come from a user.)
EVALEOF
# Add the copilot prompt inside a code fence
echo '````' >> /tmp/eval-comment.md
if [ "$SKILLS" = "yes" ]; then
cat >> /tmp/eval-comment.md <<PROMPTEOF
@copilot **Deep evaluation needed for batch results.** Read the batch aggregate results and analyze the consistently-failing tests.
Follow these steps:
1. Read \`${RESULTS_FILE}\` (the batch aggregate results on this PR's branch)
2. Read \`testing-v2/EVALUATE.md\` for the evaluation recipe
3. Read \`skills/cosmosdb-best-practices/AGENTS.md\` (all current rules)
4. Read \`testing-v2/scenarios/${SCENARIO}/api-contract.yaml\`
5. Focus on the **Consistent Failures** section — these are tests that failed in EVERY iteration and indicate real gaps
6. For each consistent failure, classify it per EVALUATE.md (Cosmos DB anti-pattern, unclear rule, SDK quirk, contract violation, test too strict)
7. For failures classified as "Cosmos DB anti-pattern" or "SDK quirk": create new rules in \`skills/cosmosdb-best-practices/rules/\`. **IMPORTANT: rules must be GENERIC, not scenario-specific.** The scenario is just one example. Write rules that apply to ANY application hitting the same Cosmos DB pattern. Use the scenario only as an illustrative example inside the rule body, never as the rule title or framing. A developer in a completely different domain should recognize the rule as relevant.
8. For failures classified as "Unclear existing rule": update the existing rule to be clearer
9. Ignore **Flaky Tests** — these are LLM stochasticity, not skill gaps
10. Run \`npm run build\` to regenerate AGENTS.md
11. Update \`testing-v2/IMPROVEMENTS-LOG.md\` with the batch evaluation results
12. Add a concise entry to \`CHANGELOG.md\` summarizing the batch evaluation and any rules created/updated
13. Commit all changes to this PR branch
PROMPTEOF
else
cat >> /tmp/eval-comment.md <<PROMPTEOF
@copilot **Deep evaluation needed for batch results (CONTROL RUN — no skills loaded).** Read the batch aggregate results and analyze what skills would have helped.
Follow these steps:
1. Read \`${RESULTS_FILE}\` (the batch aggregate results on this PR's branch)
2. Read \`testing-v2/EVALUATE.md\` for the evaluation recipe
3. Read \`skills/cosmosdb-best-practices/AGENTS.md\` (all current rules)
4. Read \`testing-v2/scenarios/${SCENARIO}/api-contract.yaml\`
5. Focus on the **Consistent Failures** section — tests that failed in EVERY iteration
6. Also review any **Build/Startup Failures** — iterations where the app failed to compile or start indicate code generation gaps
7. For each consistent failure, identify which rules from AGENTS.md WOULD HAVE helped if loaded
8. Do NOT create or update any rules — this is a control run
9. Update \`testing-v2/IMPROVEMENTS-LOG.md\` noting this was a control batch run, listing:
- Number of build/startup failures out of total iterations
- Consistent test failures and which existing rules would have prevented them
- Any failures that NO existing rule covers (these are gaps needing new rules)
10. Add a concise entry to \`CHANGELOG.md\` noting the control batch run and key findings
11. Commit all changes to this PR branch with \`[skip ci]\` in the commit message
PROMPTEOF
fi
echo '````' >> /tmp/eval-comment.md
# Strip leading whitespace from heredoc indentation
sed -i 's/^ //' /tmp/eval-comment.md
gh pr comment "$SUMMARY_PR" --body-file /tmp/eval-comment.md
echo "Posted deep evaluation prompt on summary PR #${SUMMARY_PR}"
- name: Upload aggregate artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: batch-aggregate-${{ steps.inputs.outputs.batch_issue }}
path: |
batch-results/BATCH-RESULTS.md
batch-results/batch-results.json
batch-results/run-*/test-report.json