check-models.yml

name: Sync Copilot Models

on:
  workflow_dispatch:  # Manual trigger
  push:
    paths:
        - .github/workflows/check-models.yml
        - .github/prompts/sync-models.prompt.md
        - src/tokenEstimators.json
        - src/modelPricing.json
        
  schedule:
    - cron: '11 17 * * 1' # Run every Monday at 5:11 PM UTC

permissions:
  contents: read

jobs:
  sync-models:
    runs-on: ubuntu-latest
    if: github.actor != 'dependabot[bot]'
    permissions:
      contents: write  # Need write permission to create branches and push changes
      pull-requests: write  # Need write permission to create PRs
      id-token: write  # Keep for potential future use

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@58077d3c7e43986b6b15fba718e8ea69e387dfcc # v2.15.1
        with:
          egress-policy: audit

      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0
          
      - name: Setup Node.js
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: '20.x'

      - name: Authenticate GitHub CLI
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}
        run: |
          echo "=== Authenticating GitHub CLI ==="
          if [ -z "$GH_TOKEN" ]; then
            echo "❌ Error: No authentication token available"
            echo "Environment variable GH_TOKEN is empty"
            exit 1
          fi
          
          echo ""
          echo "=== Checking authentication status ==="
          if gh auth status 2>&1; then
            echo "✅ Authentication verified"
          else
            echo "❌ Authentication check failed"
            echo "Note: This may indicate an issue with token permissions or GitHub CLI configuration"
            exit 1
          fi

      - name: Install dependencies
        run: npm ci
        
      - name: Check Copilot CLI version
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}
          COPILOT_GITHUB_TOKEN: ${{ secrets.GH_PAT }}
        run: |
          npx @github/copilot --version
        
      - name: Run Copilot CLI to update model data
        id: copilot_run
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}
          COPILOT_GITHUB_TOKEN: ${{ secrets.GH_PAT }}
        run: |
          # Read the prompt from the file
          PROMPT=$(cat .github/prompts/sync-models.prompt.md)
          
          # Get the paths to relevant files
          ESTIMATORS_PATH="${GITHUB_WORKSPACE}/src/tokenEstimators.json"
          PRICING_PATH="${GITHUB_WORKSPACE}/src/modelPricing.json"
          
          # Create the full prompt with context
          {
            echo "$PROMPT"
            echo ""
            echo "## Context Paths"
            echo ""
            echo "- Token estimators file: ${ESTIMATORS_PATH}"
            echo "- Model pricing file: ${PRICING_PATH}"
          } > /tmp/copilot-prompt.md
          
          echo "=== Running Copilot CLI with prompt ==="
          cat /tmp/copilot-prompt.md
          echo ""
          echo "=== Copilot CLI Output ==="

          cd ${GITHUB_WORKSPACE}
          
          # Run copilot with the prompt using non-interactive mode
          COPILOT_PROMPT_TEXT=$(cat /tmp/copilot-prompt.md)
          npx @github/copilot -p "$COPILOT_PROMPT_TEXT" --silent --allow-all --no-ask-user 2>&1 || {
            EXIT_CODE=$?
            echo "Warning: copilot command exited with code $EXIT_CODE, but continuing to check output"
          }
          
      - name: Check for changes
        id: changes
        run: |
          if git diff --quiet src/tokenEstimators.json src/modelPricing.json; then
            echo "No changes detected in model data files"
            echo "changed=false" >> $GITHUB_OUTPUT
          else
            # Check if the only change is the lastUpdated date in modelPricing.json
            DIFF_OUTPUT=$(git diff src/modelPricing.json)
            
            # Count the number of changed lines (lines starting with +/- but not +++ or ---)
            CHANGED_LINES=$(echo "$DIFF_OUTPUT" | grep -E '^[+-][^+-]' | wc -l)
            
            # Check if only 2 lines changed (one deletion, one addition) and both contain "lastUpdated"
            if [ "$CHANGED_LINES" -eq 2 ]; then
              # Check if all changed lines contain "lastUpdated"
              if echo "$DIFF_OUTPUT" | grep -E '^[+-][^+-]' | grep -v '"lastUpdated"' | wc -l | grep -q '^0$'; then
                # Verify no changes to tokenEstimators.json
                if git diff --quiet src/tokenEstimators.json; then
                  echo "Only lastUpdated date changed - skipping PR creation"
                  echo "changed=false" >> $GITHUB_OUTPUT
                  exit 0
                fi
              fi
            fi
            
            echo "Changes detected in model data files"
            echo "changed=true" >> $GITHUB_OUTPUT
          fi

      - name: Create Pull Request
        if: steps.changes.outputs.changed == 'true'
        uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
        with:
          branch: sync-copilot-models
          add-paths: |
            src/tokenEstimators.json
            src/modelPricing.json
          commit-message: |
            chore: sync model data with GitHub Copilot documentation
            
            This commit automatically updates src/tokenEstimators.json and
            src/modelPricing.json with new models found in the official
            GitHub Copilot documentation.
            
            Source: https://docs.github.com/en/copilot/reference/ai-models/supported-models
          title: "chore: sync model data with GitHub Copilot documentation"
          body: |
            This pull request updates model configuration files with new models
            found in the [GitHub Copilot documentation](https://docs.github.com/en/copilot/reference/ai-models/supported-models).
            
            **Updated Files:**
            - `src/tokenEstimators.json` - Character-to-token ratio estimators
            - `src/modelPricing.json` - Model pricing data
            
            **⚠️ Note:** New pricing entries are set to `0.00` by default and require manual verification.
            Please verify pricing data before merging.
            
            The changes were automatically generated using Copilot CLI to fetch and parse
            the upstream documentation and update the configuration files.
            
            ---
            
            **Workflow Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          labels: |
            autogenerated
            maintenance
        
      - name: Summary
        run: |
          if [ "${{ steps.changes.outputs.changed }}" == "true" ]; then
            echo "✅ Model data files have been successfully updated with new models from documentation"
            echo ""
            echo "A pull request has been created for review."
          else
            echo "ℹ️ Model data files were already up to date with documentation"
          fi