agentex-tutorials-test.yml

name: Test Tutorial Agents

on:
  workflow_dispatch:

jobs:
  test-tutorials:
    timeout-minutes: 15
    name: test-tutorial-${{ matrix.tutorial }}
    runs-on: ubuntu-latest

    steps:
      - name: Install UV
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.local/bin" >> $GITHUB_PATH

      - name: Checkout scale-agentex repo
        uses: actions/checkout@v4
        with:
          repository: scaleapi/scale-agentex
          path: scale-agentex

      - name: Navigate to scale-agentex repo
        run: |
          cd scale-agentex/agentex
          echo "🚀 Starting dependencies (Postgres, Redis, Temporal, MongoDB)..."

          # Start all services except the agentex service
          docker compose up -d

          echo "⏳ Waiting for dependencies to be healthy..."

          # Wait for services to be healthy
          for i in {1..30}; do
            if docker compose ps | grep -q "healthy"; then
              echo "✅ Dependencies are healthy"
              break
            fi
            echo "  Attempt $i/30: Waiting for services..."
            sleep 5
          done

          # Verify all deps are up
          docker compose ps

      - name: Run Parallel Tutorial Tests
        run: |
          cd scale-agentex/agentex/examples/tutorials

          # Find all tutorial directories
          tutorial_paths=()
          for dir in $(find . -name "manifest.yaml" -exec dirname {} \; | sort); do
            tutorial_paths+=("${dir#./}")  # Remove leading ./
          done

          echo "Found ${#tutorial_paths[@]} tutorials:"
          printf '  %s\n' "${tutorial_paths[@]}"

          # Run tests in parallel with unique ports
          pids=()
          failed_tests=()
          passed_tests=()

          for i in "${!tutorial_paths[@]}"; do
            tutorial="${tutorial_paths[$i]}"
            port=$((8000 + i))

            echo ""
            echo "========================================="
            echo "Starting test $((i+1))/${#tutorial_paths[@]}: $tutorial (port $port)"
            echo "========================================="

            # Modify manifest.yaml to use unique port
            manifest_path="$tutorial/manifest.yaml"
            if [ -f "$manifest_path" ]; then
              # Backup original manifest
              cp "$manifest_path" "$manifest_path.backup"

              # Update port in manifest (modify the line containing 'port: 8000' or 'port: XXXX')
              sed -i "s/port: [0-9]*/port: $port/" "$manifest_path"

              echo "Updated $manifest_path to use port $port"
            fi

            # Run test in background with unique port
            (
              cd "../../../agentex-python/examples/tutorials"
              AGENTEX_API_BASE_URL="http://localhost:5003" \
                ./run_agent_test.sh "../../scale-agentex/agentex/examples/tutorials/$tutorial"

              if [ $? -eq 0 ]; then
                echo "✅ PASSED: $tutorial (port $port)"
                echo "$tutorial" > "/tmp/passed_$i.txt"
              else
                echo "❌ FAILED: $tutorial (port $port)"
                echo "$tutorial" > "/tmp/failed_$i.txt"
              fi

              # Restore original manifest
              cd "../../scale-agentex/agentex/examples/tutorials"
              if [ -f "$tutorial/manifest.yaml.backup" ]; then
                mv "$tutorial/manifest.yaml.backup" "$tutorial/manifest.yaml"
              fi
            ) &

            pids+=($!)
          done

          # Wait for all tests to complete
          echo ""
          echo "Waiting for all tests to complete..."
          for pid in "${pids[@]}"; do
            wait "$pid"
          done

          # Collect results
          for i in "${!tutorial_paths[@]}"; do
            if [ -f "/tmp/passed_$i.txt" ]; then
              passed_tests+=($(cat "/tmp/passed_$i.txt"))
            elif [ -f "/tmp/failed_$i.txt" ]; then
              failed_tests+=($(cat "/tmp/failed_$i.txt"))
            fi
          done

          # Print summary
          echo ""
          echo "========================================="
          echo "TEST SUMMARY"
          echo "========================================="
          echo "Total: ${#tutorial_paths[@]}"
          echo "Passed: ${#passed_tests[@]}"
          echo "Failed: ${#failed_tests[@]}"

          if [ ${#failed_tests[@]} -gt 0 ]; then
            echo ""
            echo "Failed tests:"
            for test in "${failed_tests[@]}"; do
              echo "  ❌ $test"
            done
            exit 1
          else
            echo ""
            echo "🎉 All tests passed!"
          fi