agentex-tutorials-test.yml

name: Test Tutorial Agents

on:
  workflow_dispatch:

jobs:
  test-tutorials:
    timeout-minutes: 15
    name: test-tutorial-${{ matrix.tutorial }}
    runs-on: ubuntu-latest

    steps:
      - name: Checkout agentex-python repo
        uses: actions/checkout@v4

      - name: Install UV
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.local/bin" >> $GITHUB_PATH

      - name: Checkout scale-agentex repo
        uses: actions/checkout@v4
        with:
          repository: scaleapi/scale-agentex
          path: scale-agentex

      - name: Navigate to scale-agentex repo
        run: |
          cd scale-agentex/agentex
          echo "🚀 Starting dependencies (Postgres, Redis, Temporal, MongoDB)..."

          # Start all services
          docker compose up -d

          echo "⏳ Waiting for dependencies to be healthy..."

          # Wait for services to be healthy
          for i in {1..30}; do
            if docker compose ps | grep -q "healthy"; then
              echo "✅ Dependencies are healthy"
              break
            fi
            echo "  Attempt $i/30: Waiting for services..."
            sleep 5
          done

          # Wait specifically for AgentEx server to be ready
          echo "⏳ Waiting for AgentEx server to be ready..."
          for i in {1..30}; do
            if curl -s --max-time 5 http://localhost:5003/health >/dev/null 2>&1; then
              echo "✅ AgentEx server is ready"
              break
            fi
            echo "  Attempt $i/30: Waiting for AgentEx server..."
            sleep 5
          done

          # Debug port mappings
          echo "🔍 Docker port mappings:"
          docker compose ps
          echo ""
          echo "🔍 Docker port details:"
          docker compose ps --format "table {{.Name}}\t{{.Image}}\t{{.Ports}}"
          echo ""
          echo "🔍 Testing AgentEx server accessibility:"
          curl -v --max-time 10 http://localhost:5003/health 2>&1 || echo "❌ AgentEx not accessible on localhost:5003"

      - name: Build AgentEx SDK
        run: |
          echo "🔨 Building AgentEx SDK wheel..."
          uv build
          echo "✅ SDK built successfully"
          ls -la dist/

      - name: Run Parallel Tutorial Tests
        working-directory: ./examples/tutorials
        run: |
          # Verify uv is working
          echo "UV version: $(uv --version)"
          echo "UV path: $(which uv)"

          # Start background job to continuously poll AgentEx container logs
          echo "🔍 Starting AgentEx container log polling in background..."
          (
            echo "Log polling started at $(date)"

            # Function to get container logs
            get_logs() {
              echo "=== AgentEx Container Logs $(date) ==="
              docker logs agentex --tail=20 2>/dev/null || {
                echo "⚠️ Failed to get logs from 'agentex' container"
                echo "Available containers:"
                docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}"
              }
              echo "=== End Logs ==="
              echo ""
            }

            # Poll logs every 2 seconds (very frequent for debugging)
            while true; do
              get_logs
              sleep 2
            done
          ) &

          LOG_POLLER_PID=$!
          echo "📋 Log poller started with PID: $LOG_POLLER_PID"

          # Find all tutorial directories
          tutorial_paths=()
          for dir in $(find . -name "manifest.yaml" -exec dirname {} \; | sort); do
            tutorial_paths+=("${dir#./}")  # Remove leading ./
          done

          echo "Found ${#tutorial_paths[@]} tutorials:"
          printf '  %s\n' "${tutorial_paths[@]}"

          # Run tests in parallel with unique ports
          pids=()
          failed_tests=()
          passed_tests=()

          # Run only the first test for sanity check
          for i in 0; do
            tutorial="${tutorial_paths[$i]}"
            port=$((8000 + i))

            echo ""
            echo "========================================="
            echo "Starting test $((i+1))/${#tutorial_paths[@]}: $tutorial (port $port)"
            echo "========================================="

            # Modify manifest.yaml to use unique port
            manifest_path="$tutorial/manifest.yaml"
            if [ -f "$manifest_path" ]; then
              # Backup original manifest
              cp "$manifest_path" "$manifest_path.backup"

              # Update port in manifest (modify the line containing 'port: 8000' or 'port: XXXX')
              sed -i "s/port: [0-9]*/port: $port/" "$manifest_path"

              echo "Updated $manifest_path to use port $port"
            fi

            # Debug connectivity before running test
            echo "🔍 Pre-test connectivity check:"
            curl -v --max-time 5 http://localhost:5003/agents 2>&1 || echo "❌ /agents endpoint not accessible"

            # Run test in background with unique port
            (
              AGENTEX_API_BASE_URL="http://localhost:5003" \
                ./run_agent_test.sh --build-cli "$tutorial"

              if [ $? -eq 0 ]; then
                echo "✅ PASSED: $tutorial (port $port)"
                echo "$tutorial" > "/tmp/passed_$i.txt"
              else
                echo "❌ FAILED: $tutorial (port $port)"
                echo "$tutorial" > "/tmp/failed_$i.txt"
              fi
            ) &

            pids+=($!)
          done

          # Wait for all tests to complete
          echo ""
          echo "Waiting for all tests to complete..."
          for pid in "${pids[@]}"; do
            wait "$pid"
          done

          # Always show AgentEx server container logs immediately after tests complete
          echo ""
          echo "========================================="
          echo "AGENTEX SERVER CONTAINER LOGS"
          echo "========================================="

          # Show AgentEx server container logs
          echo "📋 AgentEx server container logs:"
          echo "----------------------------------------"

          echo "Available containers:"
          docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}"

          echo ""
          echo "AgentEx container logs (last 100 lines):"
          docker logs agentex --tail=100 2>/dev/null || {
            echo "❌ Failed to get logs from 'agentex' container"
            echo "Available containers:"
            docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}"
          }
          echo "----------------------------------------"

          # Stop the log poller
          echo "🛑 Stopping log poller (PID: $LOG_POLLER_PID)"
          kill $LOG_POLLER_PID 2>/dev/null || echo "Log poller already stopped"

          # Restore all original manifests
          echo ""
          echo "Restoring original manifest files..."
          for tutorial in "${tutorial_paths[@]}"; do
            if [ -f "$tutorial/manifest.yaml.backup" ]; then
              mv "$tutorial/manifest.yaml.backup" "$tutorial/manifest.yaml"
              echo "Restored $tutorial/manifest.yaml"
            fi
          done

          # Collect results
          for i in "${!tutorial_paths[@]}"; do
            if [ -f "/tmp/passed_$i.txt" ]; then
              passed_tests+=($(cat "/tmp/passed_$i.txt"))
            elif [ -f "/tmp/failed_$i.txt" ]; then
              failed_tests+=($(cat "/tmp/failed_$i.txt"))
            fi
          done


          # Print summary
          echo ""
          echo "========================================="
          echo "TEST SUMMARY"
          echo "========================================="
          echo "Total: ${#tutorial_paths[@]}"
          echo "Passed: ${#passed_tests[@]}"
          echo "Failed: ${#failed_tests[@]}"

          if [ ${#failed_tests[@]} -gt 0 ]; then
            echo ""
            echo "Failed tests:"
            for test in "${failed_tests[@]}"; do
              echo "  ❌ $test"
            done

            echo ""
            echo "========================================="
            echo "TUTORIAL AGENT LOGS FOR FAILED TESTS"
            echo "========================================="

            # Show logs for failed tests
            for test in "${failed_tests[@]}"; do
              test_name=$(basename "$test")
              logfile="/tmp/agentex-${test_name}.log"

              echo ""
              echo "📋 Tutorial agent logs for $test ($logfile):"
              echo "----------------------------------------"

              if [ -f "$logfile" ]; then
                echo "Last 100 lines of tutorial agent logs:"
                tail -100 "$logfile"
              else
                echo "⚠️  No log file found at: $logfile"
                echo "Available log files:"
                ls -la /tmp/agentex-*.log 2>/dev/null || echo "  (none)"
              fi

              echo "----------------------------------------"
            done

            exit 1
          else
            echo ""
            echo "🎉 All tests passed!"
          fi