Review: PR #5548 #6919
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Impl: Review" | |
| run-name: "Review: PR #${{ inputs.pr_number || github.event.client_payload.pr_number }}" | |
| # AI quality review for implementation PRs | |
| # Triggered by impl-generate.yml after PR creation | |
| # Last updated: 2025-12-23 | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pr_number: | |
| description: 'PR number to review' | |
| required: true | |
| type: string | |
| repository_dispatch: | |
| types: [review-pr] | |
| concurrency: | |
| group: impl-review-${{ inputs.pr_number || github.event.client_payload.pr_number || github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| review: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write # Needed for pushing quality score to PR branch | |
| pull-requests: write | |
| issues: write | |
| id-token: write | |
| actions: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Extract PR info | |
| id: pr | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUMBER: ${{ inputs.pr_number || github.event.client_payload.pr_number }} | |
| run: | | |
| PR_DATA=$(gh pr view "$PR_NUMBER" --json headRefName,headRefOid,body) | |
| HEAD_REF=$(echo "$PR_DATA" | jq -r '.headRefName') | |
| HEAD_SHA=$(echo "$PR_DATA" | jq -r '.headRefOid') | |
| BODY=$(echo "$PR_DATA" | jq -r '.body') | |
| # Extract spec-id and library from branch: implementation/{spec-id}/{library} | |
| SPEC_ID=$(echo "$HEAD_REF" | cut -d'/' -f2) | |
| LIBRARY=$(echo "$HEAD_REF" | cut -d'/' -f3) | |
| # Extract issue number from PR body | |
| ISSUE_NUMBER=$(echo "$BODY" | grep -oP '\*\*Parent Issue:\*\* #\K\d+' | head -1 || echo "") | |
| echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT | |
| echo "specification_id=$SPEC_ID" >> $GITHUB_OUTPUT | |
| echo "library=$LIBRARY" >> $GITHUB_OUTPUT | |
| echo "branch=$HEAD_REF" >> $GITHUB_OUTPUT | |
| echo "head_sha=$HEAD_SHA" >> $GITHUB_OUTPUT | |
| echo "issue_number=$ISSUE_NUMBER" >> $GITHUB_OUTPUT | |
| echo "::notice::Reviewing PR #$PR_NUMBER for $LIBRARY implementation of $SPEC_ID (branch: $HEAD_REF)" | |
| - name: Checkout PR code | |
| run: | | |
| git fetch origin ${{ steps.pr.outputs.head_sha }} | |
| git checkout ${{ steps.pr.outputs.head_sha }} | |
| - name: Check attempt count | |
| id: attempts | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUMBER: ${{ steps.pr.outputs.pr_number }} | |
| run: | | |
| LABELS=$(gh pr view "$PR_NUMBER" --json labels -q '.labels[].name' 2>/dev/null || echo "") | |
| if echo "$LABELS" | grep -q "ai-attempt-4"; then | |
| echo "count=4" >> $GITHUB_OUTPUT | |
| echo "display=5" >> $GITHUB_OUTPUT | |
| elif echo "$LABELS" | grep -q "ai-attempt-3"; then | |
| echo "count=3" >> $GITHUB_OUTPUT | |
| echo "display=4" >> $GITHUB_OUTPUT | |
| elif echo "$LABELS" | grep -q "ai-attempt-2"; then | |
| echo "count=2" >> $GITHUB_OUTPUT | |
| echo "display=3" >> $GITHUB_OUTPUT | |
| elif echo "$LABELS" | grep -q "ai-attempt-1"; then | |
| echo "count=1" >> $GITHUB_OUTPUT | |
| echo "display=2" >> $GITHUB_OUTPUT | |
| else | |
| echo "count=0" >> $GITHUB_OUTPUT | |
| echo "display=1" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Authenticate to GCP | |
| id: gcs | |
| continue-on-error: true | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 | |
| with: | |
| project_id: anyplot | |
| workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| - name: Setup gcloud CLI | |
| if: steps.gcs.outcome == 'success' | |
| uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3 | |
| - name: Download plot images from staging | |
| if: steps.gcs.outcome == 'success' | |
| env: | |
| SPEC_ID: ${{ steps.pr.outputs.specification_id }} | |
| LIBRARY: ${{ steps.pr.outputs.library }} | |
| LANGUAGE: python | |
| run: | | |
| mkdir -p plot_images | |
| gsutil -m cp "gs://anyplot-images/staging/${SPEC_ID}/${LANGUAGE}/${LIBRARY}/*" plot_images/ 2>/dev/null || true | |
| ls -la plot_images/ | |
| - name: Verify both theme renders exist | |
| run: | | |
| missing="" | |
| [ -f "plot_images/plot-light.png" ] || missing="${missing} plot-light.png" | |
| [ -f "plot_images/plot-dark.png" ] || missing="${missing} plot-dark.png" | |
| if [ -n "$missing" ]; then | |
| echo "::error::Missing theme render(s) in staging:${missing}" | |
| echo "::error::The impl-generate workflow must produce both plot-light.png and plot-dark.png" | |
| ls -la plot_images/ || true | |
| exit 1 | |
| fi | |
| echo "::notice::Found both theme renders — proceeding to review" | |
| - name: React with eyes emoji | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUMBER: ${{ steps.pr.outputs.pr_number }} | |
| REPOSITORY: ${{ github.repository }} | |
| run: | | |
| gh api "repos/$REPOSITORY/issues/$PR_NUMBER/reactions" -f content=eyes | |
| - name: Run AI Quality Review | |
| id: review | |
| continue-on-error: true | |
| timeout-minutes: 30 | |
| uses: anthropics/claude-code-action@ef50f123a3a9be95b60040d042717517407c7256 # v1 | |
| env: | |
| LIBRARY: ${{ steps.pr.outputs.library }} | |
| SPEC_ID: ${{ steps.pr.outputs.specification_id }} | |
| PR_NUMBER: ${{ steps.pr.outputs.pr_number }} | |
| ATTEMPT: ${{ steps.attempts.outputs.display }} | |
| with: | |
| claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} | |
| claude_args: "--model sonnet" | |
| allowed_bots: '*' | |
| prompt: | | |
| Read `prompts/workflow-prompts/ai-quality-review.md` and follow those instructions. | |
| Variables for this run: | |
| - LIBRARY: ${{ steps.pr.outputs.library }} | |
| - SPEC_ID: ${{ steps.pr.outputs.specification_id }} | |
| - PR_NUMBER: ${{ steps.pr.outputs.pr_number }} | |
| - ATTEMPT: ${{ steps.attempts.outputs.display }} | |
| - name: Extract quality score | |
| id: score | |
| if: steps.review.conclusion == 'success' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUM: ${{ steps.pr.outputs.pr_number }} | |
| run: | | |
| if [ -f "quality_score.txt" ]; then | |
| SCORE=$(cat quality_score.txt | tr -d '[:space:]') | |
| else | |
| SCORE=$(gh pr view "$PR_NUM" --json comments -q '.comments[-1].body' | grep -oP 'Score: \K\d+' | head -1 || echo "0") | |
| fi | |
| # Validate score is a number between 1-100, default to 0 if invalid | |
| if ! [[ "$SCORE" =~ ^[0-9]+$ ]] || [ "$SCORE" -lt 1 ] || [ "$SCORE" -gt 100 ]; then | |
| echo "::warning::Invalid quality score '$SCORE', defaulting to 0" | |
| SCORE="0" | |
| fi | |
| echo "score=$SCORE" >> $GITHUB_OUTPUT | |
| - name: Validate review output | |
| if: steps.review.conclusion == 'success' && steps.score.outputs.score == '0' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUM: ${{ steps.pr.outputs.pr_number }} | |
| SPEC_ID: ${{ steps.pr.outputs.specification_id }} | |
| LIBRARY: ${{ steps.pr.outputs.library }} | |
| REPOSITORY: ${{ github.repository }} | |
| RUN_ID: ${{ github.run_id }} | |
| run: | | |
| echo "::error::AI Review did not produce valid output files (score=0)" | |
| MARKER="<!-- review-retry:${SPEC_ID}:${LIBRARY} -->" | |
| # Paginate so the marker is found even on PRs with >30 comments. | |
| RETRY_COUNT=$(gh api --paginate "repos/$REPOSITORY/issues/${PR_NUM}/comments?per_page=100" \ | |
| --jq "[.[] | select(.body != null and (.body | contains(\"$MARKER\")))] | length" 2>/dev/null || echo "0") | |
| if [ "$RETRY_COUNT" -ge 1 ]; then | |
| # Already auto-retried once → final fail, require manual rerun | |
| gh pr edit "$PR_NUM" --add-label "ai-review-failed" 2>/dev/null || true | |
| gh pr comment "$PR_NUM" --body "${MARKER} | |
| ## :x: AI Review Failed (auto-retry exhausted) | |
| The AI review action completed but did not produce valid output files. Auto-retry already tried once. | |
| **What happened:** | |
| - The Claude Code Action ran | |
| - No \`quality_score.txt\` file was created | |
| **Manual rerun:** | |
| \`\`\` | |
| gh workflow run impl-review.yml -f pr_number=$PR_NUM | |
| \`\`\` | |
| --- | |
| :robot: *[impl-review](https://github.com/$REPOSITORY/actions/runs/$RUN_ID)*" | |
| exit 1 | |
| fi | |
| # First failure — post marker and auto-retry via repository_dispatch | |
| gh pr comment "$PR_NUM" --body "${MARKER} | |
| ## :wrench: AI Review Produced No Score — Auto-Retrying | |
| The Claude Code Action ran but didn't write \`quality_score.txt\`. Auto-retrying review once... | |
| --- | |
| :robot: *[impl-review](https://github.com/$REPOSITORY/actions/runs/$RUN_ID)*" | |
| gh api repos/$REPOSITORY/dispatches \ | |
| -f event_type=review-pr \ | |
| -f "client_payload[pr_number]=$PR_NUM" | |
| echo "::notice::Auto-re-triggered impl-review.yml for PR #$PR_NUM" | |
| # Mark this run as failed so the run status reflects that no verdict | |
| # was produced. The auto-retry runs in a separate workflow run. | |
| exit 1 | |
| - name: Add quality score label | |
| if: steps.review.conclusion == 'success' && steps.score.outputs.score != '0' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUM: ${{ steps.pr.outputs.pr_number }} | |
| SCORE: ${{ steps.score.outputs.score }} | |
| run: | | |
| LABEL="quality:${SCORE}" | |
| gh label create "$LABEL" --color "0e8a16" --description "Quality score ${SCORE}/100" 2>/dev/null || true | |
| gh pr edit "$PR_NUM" --add-label "$LABEL" | |
| - name: Add preliminary verdict label (early) | |
| if: steps.review.conclusion == 'success' && steps.score.outputs.score != '0' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUM: ${{ steps.pr.outputs.pr_number }} | |
| SCORE: ${{ steps.score.outputs.score }} | |
| ATTEMPT_COUNT: ${{ steps.attempts.outputs.count }} | |
| run: | | |
| # Add verdict label early to ensure it's set even if later steps fail | |
| # This is idempotent - re-running won't cause issues | |
| # Dynamic threshold based on repair attempt count: | |
| # Attempt 0 (Review 1): >= 90 | |
| # Attempt 1 (Review 2): >= 80 | |
| # Attempt 2 (Review 3): >= 70 | |
| # Attempt 3 (Review 4): >= 60 | |
| # Attempt 4 (Review 5): >= 50 | |
| THRESHOLD=$((90 - ATTEMPT_COUNT * 10)) | |
| if [ "$THRESHOLD" -lt 50 ]; then THRESHOLD=50; fi | |
| if [ "$SCORE" -ge "$THRESHOLD" ]; then | |
| echo "::notice::Adding ai-approved label (score $SCORE >= $THRESHOLD)" | |
| gh pr edit "$PR_NUM" --add-label "ai-approved" 2>/dev/null || { | |
| echo "::warning::Failed to add ai-approved label, retrying..." | |
| sleep 2 | |
| gh pr edit "$PR_NUM" --add-label "ai-approved" | |
| } | |
| else | |
| echo "::notice::Adding ai-rejected label (score $SCORE < $THRESHOLD)" | |
| gh pr edit "$PR_NUM" --add-label "ai-rejected" 2>/dev/null || { | |
| echo "::warning::Failed to add ai-rejected label, retrying..." | |
| sleep 2 | |
| gh pr edit "$PR_NUM" --add-label "ai-rejected" | |
| } | |
| if [ "$SCORE" -lt 50 ]; then | |
| gh pr edit "$PR_NUM" --add-label "quality-poor" 2>/dev/null || true | |
| fi | |
| fi | |
| - name: Install Python dependencies for metadata update | |
| if: steps.review.conclusion == 'success' && steps.score.outputs.score != '0' | |
| run: pip install pyyaml | |
| - name: Update metadata and implementation header | |
| if: steps.review.conclusion == 'success' && steps.score.outputs.score != '0' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SPEC_ID: ${{ steps.pr.outputs.specification_id }} | |
| LIBRARY: ${{ steps.pr.outputs.library }} | |
| LANGUAGE: python | |
| SCORE: ${{ steps.score.outputs.score }} | |
| BRANCH: ${{ steps.pr.outputs.branch }} | |
| run: | | |
| METADATA_FILE="plots/${SPEC_ID}/metadata/${LANGUAGE}/${LIBRARY}.yaml" | |
| IMPL_FILE="plots/${SPEC_ID}/implementations/${LANGUAGE}/${LIBRARY}.py" | |
| # Configure git auth and checkout the PR branch | |
| git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git" | |
| git fetch origin "$BRANCH" | |
| git checkout -B "$BRANCH" "origin/$BRANCH" | |
| # Update metadata file with quality score, timestamp, and review feedback | |
| if [ -f "$METADATA_FILE" ]; then | |
| TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") | |
| # Write Python script to temp file to avoid YAML/shell escaping issues | |
| cat > /tmp/update_metadata.py << 'EOF' | |
| import yaml | |
| import json | |
| import sys | |
| from pathlib import Path | |
| metadata_file = sys.argv[1] | |
| score = int(sys.argv[2]) | |
| timestamp = sys.argv[3] | |
| # Read existing review data files | |
| strengths = [] | |
| weaknesses = [] | |
| image_description = None | |
| criteria_checklist = None | |
| verdict = None | |
| impl_tags = None | |
| if Path('review_strengths.json').exists(): | |
| try: | |
| with open('review_strengths.json') as f: | |
| strengths = json.load(f) | |
| except: | |
| pass | |
| if Path('review_weaknesses.json').exists(): | |
| try: | |
| with open('review_weaknesses.json') as f: | |
| weaknesses = json.load(f) | |
| except: | |
| pass | |
| if Path('review_image_description.txt').exists(): | |
| try: | |
| with open('review_image_description.txt') as f: | |
| image_description = f.read().strip() | |
| except: | |
| pass | |
| if Path('review_checklist.json').exists(): | |
| try: | |
| with open('review_checklist.json') as f: | |
| criteria_checklist = json.load(f) | |
| except: | |
| pass | |
| if Path('review_verdict.txt').exists(): | |
| try: | |
| with open('review_verdict.txt') as f: | |
| verdict = f.read().strip() | |
| except: | |
| pass | |
| if Path('review_impl_tags.json').exists(): | |
| try: | |
| with open('review_impl_tags.json') as f: | |
| impl_tags = json.load(f) | |
| except: | |
| pass | |
| # Load existing metadata | |
| with open(metadata_file, 'r') as f: | |
| data = yaml.safe_load(f) | |
| data['quality_score'] = score | |
| data['updated'] = timestamp | |
| if 'review' not in data: | |
| data['review'] = {} | |
| # Update review section with all fields | |
| data['review']['strengths'] = strengths | |
| data['review']['weaknesses'] = weaknesses | |
| # Add extended review data (issue #2845) | |
| if image_description: | |
| data['review']['image_description'] = image_description | |
| if criteria_checklist: | |
| data['review']['criteria_checklist'] = criteria_checklist | |
| if verdict: | |
| data['review']['verdict'] = verdict | |
| # Add impl_tags (issue #2434) | |
| if impl_tags: | |
| data['impl_tags'] = impl_tags | |
| def str_representer(dumper, data): | |
| if isinstance(data, str) and data.endswith('Z') and 'T' in data: | |
| return dumper.represent_scalar('tag:yaml.org,2002:str', data, style="'") | |
| # Use literal block style for multi-line strings (image_description) | |
| if isinstance(data, str) and '\n' in data: | |
| return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') | |
| return dumper.represent_scalar('tag:yaml.org,2002:str', data) | |
| yaml.add_representer(str, str_representer) | |
| with open(metadata_file, 'w') as f: | |
| yaml.dump(data, f, default_flow_style=False, sort_keys=False, allow_unicode=True) | |
| EOF | |
| python3 /tmp/update_metadata.py "$METADATA_FILE" "$SCORE" "$TIMESTAMP" | |
| echo "::notice::Updated metadata with quality score ${SCORE} and extended review data" | |
| fi | |
| # Update implementation header with quality score | |
| if [ -f "$IMPL_FILE" ]; then | |
| # Get library and python versions from metadata | |
| LIBRARY_VERSION=$(python3 -c "import yaml; print(yaml.safe_load(open('$METADATA_FILE'))['library_version'])" 2>/dev/null || echo "unknown") | |
| PYTHON_VERSION=$(python3 -c "import yaml; print(yaml.safe_load(open('$METADATA_FILE'))['python_version'])" 2>/dev/null || echo "3.13") | |
| DATE_INFO=$(python3 -c " | |
| import yaml | |
| data = yaml.safe_load(open('$METADATA_FILE')) | |
| created = data['created'][:10] | |
| updated = data.get('updated', '')[:10] if data.get('updated') else '' | |
| if updated and updated != created: | |
| print(f'Updated: {updated}') | |
| else: | |
| print(f'Created: {created}') | |
| " 2>/dev/null || echo "Created: $(date +%Y-%m-%d)") | |
| # Get title from specification.yaml | |
| TITLE=$(python3 -c "import yaml; print(yaml.safe_load(open('plots/${SPEC_ID}/specification.yaml'))['title'])" 2>/dev/null || echo "${SPEC_ID}") | |
| # Replace old header using Python. | |
| # All inputs are passed via env (NOT shell-interpolated into the | |
| # Python source), and the heredoc uses single-quoted 'EOF' so bash | |
| # does not expand $TITLE. This blocks the prior triple-quoted | |
| # literal injection (a TITLE containing ''' would have escaped | |
| # the string and executed arbitrary Python in the runner). | |
| export IMPL_FILE SPEC_ID TITLE LIBRARY LIBRARY_VERSION PYTHON_VERSION SCORE DATE_INFO | |
| python3 - <<'EOF' | |
| import os | |
| import re | |
| impl_file = os.environ["IMPL_FILE"] | |
| spec_id = os.environ["SPEC_ID"] | |
| title = os.environ["TITLE"] | |
| library = os.environ["LIBRARY"] | |
| lib_version = os.environ["LIBRARY_VERSION"] | |
| py_version = os.environ["PYTHON_VERSION"] | |
| score = os.environ["SCORE"] | |
| date_info = os.environ["DATE_INFO"] | |
| # Sanitize title to prevent triple-quote injection | |
| title = title.replace('"""', '\"\"\"') | |
| new_header = f'''""" anyplot.ai | |
| {spec_id}: {title} | |
| Library: {library} {lib_version} | Python {py_version} | |
| Quality: {score}/100 | {date_info} | |
| """''' | |
| with open(impl_file, "r") as f: | |
| content = f.read() | |
| pattern = r'^""".*?"""' | |
| new_content = re.sub(pattern, new_header, content, count=1, flags=re.DOTALL) | |
| with open(impl_file, "w") as f: | |
| f.write(new_content) | |
| EOF | |
| echo "::notice::Updated implementation header with quality score ${SCORE}" | |
| fi | |
| # Commit and push | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add "$METADATA_FILE" "$IMPL_FILE" | |
| if ! git diff --cached --quiet; then | |
| git commit -m "chore(${LIBRARY}): update quality score ${SCORE} and review feedback for ${SPEC_ID}" | |
| git push origin "$BRANCH" | |
| echo "::notice::Changes committed to ${BRANCH}" | |
| fi | |
| - name: Handle review failure | |
| if: steps.attempts.outputs.count != '4' && steps.review.conclusion == 'failure' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUM: ${{ steps.pr.outputs.pr_number }} | |
| SPEC_ID: ${{ steps.pr.outputs.specification_id }} | |
| LIBRARY: ${{ steps.pr.outputs.library }} | |
| REPOSITORY: ${{ github.repository }} | |
| RUN_ID: ${{ github.run_id }} | |
| run: | | |
| MARKER="<!-- review-retry:${SPEC_ID}:${LIBRARY} -->" | |
| # Paginate so the marker is found even on PRs with >30 comments. | |
| RETRY_COUNT=$(gh api --paginate "repos/$REPOSITORY/issues/${PR_NUM}/comments?per_page=100" \ | |
| --jq "[.[] | select(.body != null and (.body | contains(\"$MARKER\")))] | length" 2>/dev/null || echo "0") | |
| if [ "$RETRY_COUNT" -ge 1 ]; then | |
| gh pr edit "$PR_NUM" --add-label "ai-review-failed" 2>/dev/null || true | |
| gh pr comment "$PR_NUM" --body "${MARKER} | |
| ## :x: AI Review Failed (auto-retry exhausted) | |
| The AI review action failed or timed out twice in a row. | |
| **Manual rerun:** | |
| \`\`\` | |
| gh workflow run impl-review.yml -f pr_number=$PR_NUM | |
| \`\`\` | |
| --- | |
| :robot: *[impl-review](https://github.com/$REPOSITORY/actions/runs/$RUN_ID)*" | |
| exit 0 | |
| fi | |
| gh pr comment "$PR_NUM" --body "${MARKER} | |
| ## :wrench: AI Review Crashed — Auto-Retrying | |
| The Claude Code Action failed or timed out. Auto-retrying review once... | |
| --- | |
| :robot: *[impl-review](https://github.com/$REPOSITORY/actions/runs/$RUN_ID)*" | |
| gh api repos/$REPOSITORY/dispatches \ | |
| -f event_type=review-pr \ | |
| -f "client_payload[pr_number]=$PR_NUM" | |
| echo "::notice::Auto-re-triggered impl-review.yml for PR #$PR_NUM" | |
| # Mark this run as failed so the run status reflects that no verdict | |
| # was produced. The auto-retry runs in a separate workflow run. | |
| exit 1 | |
| - name: Add verdict label and take action | |
| if: steps.review.conclusion == 'success' && steps.score.outputs.score != '0' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUM: ${{ steps.pr.outputs.pr_number }} | |
| SPEC_ID: ${{ steps.pr.outputs.specification_id }} | |
| LIBRARY: ${{ steps.pr.outputs.library }} | |
| LANGUAGE: python | |
| SCORE: ${{ steps.score.outputs.score }} | |
| ATTEMPT: ${{ steps.attempts.outputs.display }} | |
| ATTEMPT_COUNT: ${{ steps.attempts.outputs.count }} | |
| ISSUE_NUMBER: ${{ steps.pr.outputs.issue_number }} | |
| REPOSITORY: ${{ github.repository }} | |
| RUN_ID: ${{ github.run_id }} | |
| run: | | |
| # Cascading thresholds: | |
| # Attempt 0 (Review 1): >= 90 | |
| # Attempt 1 (Review 2): >= 80 | |
| # Attempt 2 (Review 3): >= 70 | |
| # Attempt 3 (Review 4): >= 60 | |
| # Attempt 4 (Review 5): >= 50 | |
| THRESHOLD=$((90 - ATTEMPT_COUNT * 10)) | |
| if [ "$THRESHOLD" -lt 50 ]; then THRESHOLD=50; fi | |
| # Check if verdict label was already added by earlier step | |
| CURRENT_LABELS=$(gh pr view "$PR_NUM" --json labels -q '.labels[].name' 2>/dev/null || echo "") | |
| if echo "$CURRENT_LABELS" | grep -q "ai-approved"; then | |
| echo "::notice::ai-approved label already set (Score $SCORE >= $THRESHOLD), triggering merge" | |
| gh workflow run impl-merge.yml -f pr_number="$PR_NUM" | |
| exit 0 | |
| fi | |
| if echo "$CURRENT_LABELS" | grep -q "ai-rejected"; then | |
| # Still have repair attempts left? (Max 4 repairs = 5 reviews total) | |
| if [ "$ATTEMPT_COUNT" -lt 4 ]; then | |
| echo "::notice::ai-rejected label set (Score $SCORE < $THRESHOLD), triggering repair attempt $((ATTEMPT_COUNT + 1))" | |
| gh pr edit "$PR_NUM" --add-label "ai-attempt-${ATTEMPT}" 2>/dev/null || true | |
| gh workflow run impl-repair.yml \ | |
| -f pr_number="$PR_NUM" \ | |
| -f specification_id="$SPEC_ID" \ | |
| -f library="$LIBRARY" \ | |
| -f attempt="$ATTEMPT" | |
| else | |
| # All 4 repair attempts exhausted | |
| echo "::notice::All 4 repair attempts exhausted (Score $SCORE < 50)" | |
| gh pr edit "$PR_NUM" --add-label "quality-poor" | |
| gh pr comment "$PR_NUM" --body "$(cat <<'EOF' | |
| ## AI Review - Final Status | |
| ### Score: $SCORE/100 (Below Threshold) | |
| After **4 repair attempts**, the score ($SCORE) is still below the minimum acceptable threshold of 50. | |
| **This is treated like an auto-reject.** The PR will be closed and any old implementation will be removed from main. | |
| **Options:** | |
| 1. Regenerate from scratch with `generate:$LIBRARY` label | |
| 2. Manual complete rewrite | |
| --- | |
| :robot: *[impl-review](https://github.com/$REPOSITORY/actions/runs/$RUN_ID)* | |
| EOF | |
| )" | |
| gh pr close "$PR_NUM" | |
| # Remove old implementation from main if it exists | |
| IMPL_FILE="plots/${SPEC_ID}/implementations/${LANGUAGE}/${LIBRARY}.py" | |
| META_FILE="plots/${SPEC_ID}/metadata/${LANGUAGE}/${LIBRARY}.yaml" | |
| git fetch origin main | |
| if git show origin/main:"$IMPL_FILE" &>/dev/null; then | |
| echo "::notice::Removing old implementation from main: $IMPL_FILE" | |
| git checkout main | |
| git pull origin main | |
| # Delete implementation and metadata | |
| rm -f "$IMPL_FILE" "$META_FILE" | |
| # Commit and push | |
| git add -A | |
| git commit -m "chore(${LIBRARY}): remove ${SPEC_ID} impl (score ${SCORE} < 50 after 4 attempts)" | |
| git push origin main | |
| echo "::notice::Old implementation removed from main" | |
| fi | |
| if [ -n "$ISSUE_NUMBER" ]; then | |
| gh issue edit "$ISSUE_NUMBER" --add-label "impl:${LIBRARY}:failed" 2>/dev/null || true | |
| gh issue comment "$ISSUE_NUMBER" --body "**${LIBRARY}** implementation: score ${SCORE}/100 after 4 repair attempts (< 50 = rejected). PR #${PR_NUM} closed. Old implementation removed from repository." | |
| fi | |
| fi | |
| exit 0 | |
| fi |