test-providers.yml

name: Test LLM Providers

on:
  # Run on manual trigger only
  workflow_dispatch:

permissions:
  contents: write

jobs:
  test-providers:
    runs-on: ubuntu-latest

    env:
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      CODESTRAL_API_KEY: ${{ secrets.CODESTRAL_API_KEY }}
      DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
      GOOGLE_FREE_API_KEY: ${{ secrets.GOOGLE_FREE_API_KEY }}
      GROK_API_KEY: ${{ secrets.GROK_API_KEY }}
      GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
      MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
      ZAI_API_KEY: ${{ secrets.ZAI_API_KEY }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.x"
      
      - name: Install llms-py package
        run: |
          python -m pip install --upgrade pip
          pip install llms-py
      
      - name: Verify installation
        run: |
          llms
      
      - name: Test providers
        id: test-providers
        continue-on-error: true
        run: |
          # Create output directory
          mkdir -p test-results
          
          # Initialize results file
          RESULTS_TXT="test-results/provider-test-results.txt"

          echo "LLM Provider Test Results" > $RESULTS_TXT
          echo "=========================" >> $RESULTS_TXT
          echo "Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> $RESULTS_TXT
          echo "llms-py version: $(pip show llms-py | grep Version | cut -d' ' -f2)" >> $RESULTS_TXT
          echo "" >> $RESULTS_TXT
          
          # List of providers to test
          PROVIDERS=(
            "openrouter_free"
            "groq"
            "codestral"
            "openrouter"
            "google"
            "anthropic"
            "openai"
            "grok"
            "qwen"
            "z.ai"
            "qwen"
            "mistral"
          )
          
          # Test each provider
          for provider in "${PROVIDERS[@]}"; do
            echo "Testing provider: $provider" >> $RESULTS_TXT
            echo "-----------------------------------" >> $RESULTS_TXT
            
            # Run the check command and capture output
            if output=$(llms --check "$provider" 2>&1); then
              echo "Status: PASS" >> $RESULTS_TXT
              echo "$output" >> $RESULTS_TXT
            else
              echo "Status: FAIL" >> $RESULTS_TXT
              echo "$output" >> $RESULTS_TXT
            fi
            
            echo "" >> $RESULTS_TXT
            echo "" >> $RESULTS_TXT
          done
          
          echo "=========================" >> $RESULTS_TXT
          echo "Test run completed" >> $RESULTS_TXT

          # Display results in console
          echo ""
          echo "=========================================="
          echo "Test Results Summary"
          echo "=========================================="
          cat $RESULTS_TXT
          echo "=========================================="
      
      - name: Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: provider-test-results
          path: test-results/provider-test-results.txt
          retention-days: 90

      - name: Save results to repository
        if: always()
        run: |
          # Create docs/checks directory if it doesn't exist
          mkdir -p docs/checks

          # Get current date in YYYY-MM-DD format
          DATE=$(date -u '+%Y-%m-%d')

          # Copy results to both latest.txt and dated file
          cp test-results/provider-test-results.txt docs/checks/latest.txt
          cp test-results/provider-test-results.txt docs/checks/${DATE}.txt

          echo "Results saved to:"
          echo "  - docs/checks/latest.txt"
          echo "  - docs/checks/${DATE}.txt"

      - name: Commit and push results
        if: always()
        run: |
          git config --local user.email "github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"

          git add docs/checks/

          # Check if there are changes to commit
          if git diff --staged --quiet; then
            echo "No changes to commit"
          else
            DATE=$(date -u '+%Y-%m-%d')
            git commit -m "Update LLM provider test results - ${DATE}"
            git push
            echo "Results committed and pushed successfully"
          fi