agentex-tutorials-test.yml

name: Test Tutorial Agents

on:
  workflow_dispatch:

jobs:
  find-tutorials:
    runs-on: ubuntu-latest
    outputs:
      tutorials: ${{ steps.get-tutorials.outputs.tutorials }}
    steps:
      - name: Checkout agentex-python repo
        uses: actions/checkout@v4

      - name: Find all tutorials
        id: get-tutorials
        run: |
          cd examples/tutorials
          tutorials=$(find . -name "manifest.yaml" -exec dirname {} \; | sort | sed 's|^\./||' | jq -R -s -c 'split("\n") | map(select(length > 0))')
          echo "tutorials=$tutorials" >> $GITHUB_OUTPUT
          echo "Found tutorials: $tutorials"

  test-tutorial:
    needs: find-tutorials
    runs-on: ubuntu-latest
    timeout-minutes: 15
    strategy:
      matrix:
        tutorial: ${{ fromJson(needs.find-tutorials.outputs.tutorials) }}
      fail-fast: false
    name: test-${{ matrix.tutorial }}

    steps:
      - name: Checkout agentex-python repo
        uses: actions/checkout@v4

      - name: Install UV
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.local/bin" >> $GITHUB_PATH

      - name: Checkout scale-agentex repo
        uses: actions/checkout@v4
        with:
          repository: scaleapi/scale-agentex
          path: scale-agentex

      - name: Configure Docker Compose for host networking
        run: |
          cd scale-agentex/agentex
          echo "🔧 Configuring AgentEx container for GitHub Actions networking..."

          # Install yq for YAML manipulation
          sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
          sudo chmod +x /usr/local/bin/yq

          # Add extra_hosts to agentex service to make host.docker.internal work
          yq eval '.services.agentex.extra_hosts = ["host.docker.internal:host-gateway"]' -i docker-compose.yml

          echo "✅ Added extra_hosts configuration to agentex service"

      - name: Start AgentEx Server
        run: |
          cd scale-agentex/agentex
          echo "🚀 Starting AgentEx server and dependencies..."

          # Start all services
          docker compose up -d

          echo "⏳ Waiting for dependencies to be healthy..."

          # Wait for services to be healthy
          for i in {1..30}; do
            if docker compose ps | grep -q "healthy"; then
              echo "✅ Dependencies are healthy"
              break
            fi
            echo "  Attempt $i/30: Waiting for services..."
            sleep 5
          done

          # Wait specifically for AgentEx server to be ready
          echo "⏳ Waiting for AgentEx server to be ready..."
          for i in {1..30}; do
            if curl -s --max-time 5 http://localhost:5003/health >/dev/null 2>&1; then
              echo "✅ AgentEx server is ready"
              break
            fi
            echo "  Attempt $i/30: Waiting for AgentEx server..."
            sleep 5
          done

      - name: Build AgentEx SDK
        run: |
          echo "🔨 Building AgentEx SDK wheel..."
          uv build
          echo "✅ SDK built successfully"
          ls -la dist/

      - name: Test Tutorial
        working-directory: ./examples/tutorials
        env:
          OPENAI_API_KEY: ${{ secrets.TUTORIAL_OPENAI_API_KEY }}
          HEALTH_CHECK_PORT: 8080  # Use non-privileged port for temporal worker health checks
        run: |
          echo "Testing tutorial: ${{ matrix.tutorial }}"
          AGENTEX_API_BASE_URL="http://localhost:5003" \
            ./run_agent_test.sh --build-cli "${{ matrix.tutorial }}"

      - name: Upload Test Results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: test-results-${{ replace(matrix.tutorial, '/', '-') }}
          path: |
            /tmp/agentex-*.log
          retention-days: 1

  test-summary:
    if: always()
    needs: [find-tutorials, test-tutorial]
    runs-on: ubuntu-latest
    name: Test Summary
    steps:
      - name: Download All Test Results
        uses: actions/download-artifact@v4
        with:
          path: test-results
          pattern: test-results-*

      - name: Generate Test Summary
        run: |
          echo "# 🧪 Tutorial Tests Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          # Get tutorial list from needs context
          tutorials='${{ needs.find-tutorials.outputs.tutorials }}'

          # Initialize counters
          total_tutorials=0
          passed_tutorials=0
          failed_tutorials=0

          # Arrays to track results
          passed_tests=()
          failed_tests=()

          echo "## 📊 Overall Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          # Process each tutorial result
          for tutorial_dir in test-results/test-results-*/; do
            if [ -d "$tutorial_dir" ]; then
              # Extract sanitized name and convert back to original tutorial path
              sanitized_name=$(basename "$tutorial_dir" | sed 's/test-results-//')
              tutorial_name=$(echo "$sanitized_name" | sed 's/-/\//g')
              total_tutorials=$((total_tutorials + 1))

              # Determine success/failure based on presence of error logs or patterns
              if find "$tutorial_dir" -name "*.log" -exec grep -l "FAILED\|ERROR\|Traceback" {} \; | head -1 >/dev/null; then
                failed_tutorials=$((failed_tutorials + 1))
                failed_tests+=("$tutorial_name")
              else
                passed_tutorials=$((passed_tutorials + 1))
                passed_tests+=("$tutorial_name")
              fi
            fi
          done

          # Show summary stats
          echo "| Status | Count |" >> $GITHUB_STEP_SUMMARY
          echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY
          echo "| ✅ **Passed** | **$passed_tutorials** |" >> $GITHUB_STEP_SUMMARY
          echo "| ❌ **Failed** | **$failed_tutorials** |" >> $GITHUB_STEP_SUMMARY
          echo "| 📊 **Total** | **$total_tutorials** |" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          # Show passed tests
          if [ $passed_tutorials -gt 0 ]; then
            echo "## ✅ Passed Tutorials ($passed_tutorials)" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            for test in "${passed_tests[@]}"; do
              echo "- ✅ \`$test\`" >> $GITHUB_STEP_SUMMARY
            done
            echo "" >> $GITHUB_STEP_SUMMARY
          fi

          # Show failed tests with details
          if [ $failed_tutorials -gt 0 ]; then
            echo "## ❌ Failed Tutorials ($failed_tutorials)" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY

            for test in "${failed_tests[@]}"; do
              echo "### 🔍 \`$test\`" >> $GITHUB_STEP_SUMMARY
              echo "" >> $GITHUB_STEP_SUMMARY

              # Find the log file for this test (convert back to sanitized name)
              sanitized_test_name=$(echo "$test" | sed 's/\//-/g')
              log_file=$(find "test-results/test-results-$sanitized_test_name" -name "*.log" | head -1)
              if [ -f "$log_file" ]; then
                # Extract pytest failures
                if grep -q "FAILED\|ERROR" "$log_file"; then
                  echo "**Failed Tests:**" >> $GITHUB_STEP_SUMMARY
                  echo '```' >> $GITHUB_STEP_SUMMARY
                  grep -A 5 -B 1 "FAILED\|ERROR" "$log_file" | head -20 >> $GITHUB_STEP_SUMMARY
                  echo '```' >> $GITHUB_STEP_SUMMARY
                  echo "" >> $GITHUB_STEP_SUMMARY
                fi

                # Show any Python tracebacks
                if grep -q "Traceback" "$log_file"; then
                  echo "**Error Details:**" >> $GITHUB_STEP_SUMMARY
                  echo '```' >> $GITHUB_STEP_SUMMARY
                  # Get the last traceback in the file
                  awk '/Traceback \(most recent call last\)/{p=1} p{print} /^[^ ]/ && p && !/Traceback/{p=0}' "$log_file" | tail -20 >> $GITHUB_STEP_SUMMARY
                  echo '```' >> $GITHUB_STEP_SUMMARY
                  echo "" >> $GITHUB_STEP_SUMMARY
                fi
              else
                echo "_No log file found for detailed error analysis_" >> $GITHUB_STEP_SUMMARY
                echo "" >> $GITHUB_STEP_SUMMARY
              fi
            done
          fi

          # Set exit code based on results
          if [ $failed_tutorials -gt 0 ]; then
            echo "❌ Some tutorials failed. Check the details above." >> $GITHUB_STEP_SUMMARY
            exit 1
          else
            echo "🎉 All tutorials passed successfully!" >> $GITHUB_STEP_SUMMARY
          fi