agentex-tutorials-test.yml

name: Test Tutorial Agents

on:
  workflow_dispatch:

jobs:
  find-tutorials:
    runs-on: ubuntu-latest
    outputs:
      tutorials: ${{ steps.get-tutorials.outputs.tutorials }}
    steps:
      - name: Checkout agentex-python repo
        uses: actions/checkout@v4

      - name: Find all tutorials
        id: get-tutorials
        run: |
          cd examples/tutorials
          # Find all tutorials and exclude specific temporal ones
          all_tutorials=$(find . -name "manifest.yaml" -exec dirname {} \; | sort | sed 's|^\./||')

          # Filter out the specified temporal tutorials that are being updated
          filtered_tutorials=$(echo "$all_tutorials" | grep -v -E "(temporal)")

          # Convert to JSON array
          tutorials=$(echo "$filtered_tutorials" | jq -R -s -c 'split("\n") | map(select(length > 0))')

          echo "tutorials=$tutorials" >> $GITHUB_OUTPUT
          echo "All tutorials found: $(echo "$all_tutorials" | wc -l)"
          echo "Filtered tutorials: $(echo "$filtered_tutorials" | wc -l)"
          echo "Excluded tutorials:"
          echo "$all_tutorials" | grep -E "(10_temporal/050_|10_temporal/070_|10_temporal/080_)" || echo "  (none matched exclusion pattern)"
          echo "Final tutorial list: $tutorials"

  test-tutorial:
    needs: find-tutorials
    runs-on: ubuntu-latest
    timeout-minutes: 15
    strategy:
      matrix:
        tutorial: ${{ fromJson(needs.find-tutorials.outputs.tutorials) }}
      fail-fast: false
    name: test-${{ matrix.tutorial }}

    steps:
      - name: Checkout agentex-python repo
        uses: actions/checkout@v4

      - name: Install UV
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.local/bin" >> $GITHUB_PATH

      - name: Checkout scale-agentex repo
        uses: actions/checkout@v4
        with:
          repository: scaleapi/scale-agentex
          path: scale-agentex

      - name: Configure Docker Compose for host networking
        run: |
          cd scale-agentex/agentex
          echo "🔧 Configuring AgentEx container for GitHub Actions networking..."

          # Install yq for YAML manipulation
          sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
          sudo chmod +x /usr/local/bin/yq

          # Add extra_hosts to agentex service to make host.docker.internal work
          yq eval '.services.agentex.extra_hosts = ["host.docker.internal:host-gateway"]' -i docker-compose.yml

          echo "✅ Added extra_hosts configuration to agentex service"

      - name: Start AgentEx Server
        run: |
          cd scale-agentex/agentex
          echo "🚀 Starting AgentEx server and dependencies..."

          # Start all services
          docker compose up -d

          echo "⏳ Waiting for dependencies to be healthy..."

          # Wait for services to be healthy
          for i in {1..30}; do
            if docker compose ps | grep -q "healthy"; then
              echo "✅ Dependencies are healthy"
              break
            fi
            echo "  Attempt $i/30: Waiting for services..."
            sleep 5
          done

          # Wait specifically for AgentEx server to be ready
          echo "⏳ Waiting for AgentEx server to be ready..."
          for i in {1..30}; do
            if curl -s --max-time 5 http://localhost:5003/health >/dev/null 2>&1; then
              echo "✅ AgentEx server is ready"
              break
            fi
            echo "  Attempt $i/30: Waiting for AgentEx server..."
            sleep 5
          done

      - name: Build AgentEx SDK
        run: |
          echo "🔨 Building AgentEx SDK wheel..."
          uv build
          echo "✅ SDK built successfully"
          ls -la dist/

      - name: Test Tutorial
        working-directory: ./examples/tutorials
        env:
          OPENAI_API_KEY: ${{ secrets.TUTORIAL_OPENAI_API_KEY }}
          HEALTH_CHECK_PORT: 8080 # Use non-privileged port for temporal worker health checks
        run: |
          echo "Testing tutorial: ${{ matrix.tutorial }}"

          # Create output files
          SANITIZED_NAME=$(echo "${{ matrix.tutorial }}" | sed 's/\//-/g')
          OUTPUT_FILE="/tmp/test-output-$SANITIZED_NAME.txt"
          STATUS_FILE="/tmp/test-status-$SANITIZED_NAME.txt"

          # Run the test, show output live, and also capture to file
          set +e  # Don't exit on error
          AGENTEX_API_BASE_URL="http://localhost:5003" \
            ./run_agent_test.sh --build-cli "${{ matrix.tutorial }}" 2>&1 | tee "$OUTPUT_FILE"
          EXIT_CODE=${PIPESTATUS[0]}  # Get exit code from the first command in pipeline
          set -e  # Re-enable exit on error

          # Write the exit code to status file
          echo "$EXIT_CODE" > "$STATUS_FILE"

          # Exit with the original exit code to make the job succeed/fail correctly
          exit $EXIT_CODE

      - name: Upload Test Results
        if: always()
        run: |
          # Sanitize tutorial name for artifact upload
          SANITIZED_NAME=$(echo "${{ matrix.tutorial }}" | sed 's/\//-/g')
          echo "Uploading test results for: ${{ matrix.tutorial }} (as: test-results-$SANITIZED_NAME)"

          # Create a temporary directory with the sanitized name
          mkdir -p "test-results-$SANITIZED_NAME"

          # Copy the test output and status files
          cp "/tmp/test-output-$SANITIZED_NAME.txt" "test-results-$SANITIZED_NAME/" 2>/dev/null || echo "No output file to copy"
          cp "/tmp/test-status-$SANITIZED_NAME.txt" "test-results-$SANITIZED_NAME/" 2>/dev/null || echo "No status file to copy"

          # Also copy agent logs if they exist
          cp /tmp/agentex-*.log "test-results-$SANITIZED_NAME/" 2>/dev/null || echo "No agent log files to copy"

          # Upload using the actions/upload-artifact action
          echo "artifact-name=test-results-$SANITIZED_NAME" >> $GITHUB_ENV

      - name: Upload Artifact
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.artifact-name }}
          path: test-results-*
          retention-days: 1

  test-summary:
    if: always()
    needs: [find-tutorials, test-tutorial]
    runs-on: ubuntu-latest
    name: Test Summary
    steps:
      - name: Download All Test Results
        uses: actions/download-artifact@v4
        with:
          path: test-results
          pattern: test-results-*

      - name: Generate Test Summary
        run: |
          echo "# 🧪 Tutorial Tests Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          # Get tutorial list from needs context
          tutorials='${{ needs.find-tutorials.outputs.tutorials }}'

          # Initialize counters
          total_tutorials=0
          passed_tutorials=0
          failed_tutorials=0

          # Arrays to track results
          passed_tests=()
          failed_tests=()

          echo "## 📊 Overall Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          # Process each tutorial and check its status file
          echo "$tutorials" | jq -r '.[]' | while read -r tutorial_name; do
            total_tutorials=$((total_tutorials + 1))

            # Convert tutorial name to sanitized format
            sanitized_name=$(echo "$tutorial_name" | sed 's/\//-/g')
            tutorial_dir="test-results/test-results-$sanitized_name"
            status_file="$tutorial_dir/test-status-$sanitized_name.txt"

            if [ -f "$status_file" ]; then
              exit_code=$(cat "$status_file")
              if [ "$exit_code" = "0" ]; then
                passed_tutorials=$((passed_tutorials + 1))
                passed_tests+=("$tutorial_name")
                echo "✅ $tutorial_name: PASSED (exit code: $exit_code)"
              else
                failed_tutorials=$((failed_tutorials + 1))
                failed_tests+=("$tutorial_name")
                echo "❌ $tutorial_name: FAILED (exit code: $exit_code)"
              fi
            else
              # No status file found, assume failed
              failed_tutorials=$((failed_tutorials + 1))
              failed_tests+=("$tutorial_name")
              echo "❌ $tutorial_name: FAILED (no status file)"
            fi
          done

          # Show summary stats
          echo "| Status | Count |" >> $GITHUB_STEP_SUMMARY
          echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY
          echo "| ✅ **Passed** | **$passed_tutorials** |" >> $GITHUB_STEP_SUMMARY
          echo "| ❌ **Failed** | **$failed_tutorials** |" >> $GITHUB_STEP_SUMMARY
          echo "| 📊 **Total** | **$total_tutorials** |" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          # Show passed tests
          if [ $passed_tutorials -gt 0 ]; then
            echo "## ✅ Passed Tutorials ($passed_tutorials)" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            for test in "${passed_tests[@]}"; do
              echo "- ✅ \`$test\`" >> $GITHUB_STEP_SUMMARY
            done
            echo "" >> $GITHUB_STEP_SUMMARY
          fi

          # Show test output for failed tests
          if [ $failed_tutorials -gt 0 ]; then
            echo "## ❌ Failed Tutorials ($failed_tutorials)" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            echo '```' >> $GITHUB_STEP_SUMMARY

            # Show the full output for each failed test
            for test in "${failed_tests[@]}"; do
              sanitized_test_name=$(echo "$test" | sed 's/\//-/g')
              output_file="test-results/test-results-$sanitized_test_name/test-output-$sanitized_test_name.txt"

              if [ -f "$output_file" ]; then
                echo "================================================================================================" >> $GITHUB_STEP_SUMMARY
                echo "FAILED: $test" >> $GITHUB_STEP_SUMMARY
                echo "================================================================================================" >> $GITHUB_STEP_SUMMARY
                cat "$output_file" >> $GITHUB_STEP_SUMMARY
                echo "" >> $GITHUB_STEP_SUMMARY
              else
                echo "================================================================================================" >> $GITHUB_STEP_SUMMARY
                echo "FAILED: $test (No output file found)" >> $GITHUB_STEP_SUMMARY
                echo "================================================================================================" >> $GITHUB_STEP_SUMMARY
                echo "" >> $GITHUB_STEP_SUMMARY
              fi
            done

            echo '```' >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
          fi

          # Set exit code based on results
          if [ $failed_tutorials -gt 0 ]; then
            echo "❌ Some tutorials failed. Check the details above." >> $GITHUB_STEP_SUMMARY
            exit 1
          else
            echo "🎉 All tutorials passed successfully!" >> $GITHUB_STEP_SUMMARY
          fi