e2e-autotest.yml

name: E2E AutoTest

on:
  workflow_dispatch:
    inputs:
      test_plan:
        description: "Test plan to run (leave empty for all)"
        required: false
        default: ""
        type: string
      vsix_urls:
        description: "VSIX URLs to install, comma-separated (e.g. https://host/redhat.java-1.42.0.vsix,https://host/vscode-java-debug-0.58.0.vsix)"
        required: false
        default: ""
        type: string

jobs:
  # ── Job 1: Discover test plans ──────────────────────────
  discover:
    if: ${{ inputs.test_plan == '' }}
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.scan.outputs.matrix }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Scan test plans
        id: scan
        run: |
          plans=$(ls test-plans/*.yaml | xargs -I{} basename {} .yaml | grep -v java-fresh-import | jq -R . | jq -sc .)
          echo "matrix=$plans" >> "$GITHUB_OUTPUT"
          echo "Found plans: $plans"

  # ── Job 2: Run each test plan in parallel ───────────────
  e2e-test:
    if: ${{ inputs.test_plan == '' }}
    needs: discover
    runs-on: windows-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        plan: ${{ fromJson(needs.discover.outputs.matrix) }}

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Clone vscode-java (test projects)
        run: git clone --depth 1 https://github.com/redhat-developer/vscode-java.git ../vscode-java

      - name: Clone eclipse.jdt.ls (Gradle test projects)
        run: git clone --depth 1 https://github.com/eclipse-jdtls/eclipse.jdt.ls.git ../eclipse.jdt.ls

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Setup Java 25 (for java25 test plans)
        if: contains(matrix.plan, 'java25')
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: 25-ea

      - name: Create JDK 25 path
        if: contains(matrix.plan, 'java25')
        shell: pwsh
        run: |
          New-Item -ItemType Junction -Path "C:\Program Files\Java\jdk-25" -Target $env:JAVA_HOME

      - name: Setup Java 21
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: 21

      - name: Install autotest CLI
        run: npm install -g @vscjava/vscode-autotest

      - name: Download VSIX files
        if: ${{ inputs.vsix_urls != '' }}
        shell: pwsh
        run: |
          New-Item -ItemType Directory -Path vsix -Force | Out-Null
          $urls = "${{ inputs.vsix_urls }}" -split "," | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" }
          foreach ($url in $urls) {
            $fileName = [System.IO.Path]::GetFileName(($url -split '\?')[0])
            Write-Host "Downloading: $url → vsix/$fileName"
            Invoke-WebRequest -Uri $url -OutFile "vsix/$fileName" -UseBasicParsing
          }
          Write-Host "Downloaded VSIX files:"
          Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { Write-Host "  $($_.Name) ($([math]::Round($_.Length/1MB, 1)) MB)" }

      - name: Run ${{ matrix.plan }}
        shell: pwsh
        env:
          AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }}
        run: |
          $autotestArgs = @("run", "test-plans/${{ matrix.plan }}.yaml")
          if (Test-Path vsix) {
            $vsixFiles = (Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { $_.FullName }) -join ","
            if ($vsixFiles) { $autotestArgs += @("--vsix", $vsixFiles) }
          }
          Write-Host "Running: autotest $($autotestArgs -join ' ')"
          & autotest @autotestArgs

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: results-${{ matrix.plan }}
          path: test-results/
          retention-days: 30

  # ── Job 2b: Run a single test plan (when specified) ─────
  e2e-single:
    if: ${{ inputs.test_plan != '' }}
    runs-on: windows-latest
    timeout-minutes: 30

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Clone vscode-java
        run: git clone --depth 1 https://github.com/redhat-developer/vscode-java.git ../vscode-java

      - name: Clone eclipse.jdt.ls
        run: git clone --depth 1 https://github.com/eclipse-jdtls/eclipse.jdt.ls.git ../eclipse.jdt.ls

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Setup Java 25
        if: contains(inputs.test_plan, 'java25')
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: 25-ea

      - name: Create JDK 25 path
        if: contains(inputs.test_plan, 'java25')
        shell: pwsh
        run: |
          New-Item -ItemType Junction -Path "C:\Program Files\Java\jdk-25" -Target $env:JAVA_HOME

      - name: Setup Java 21
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: 21

      - name: Install autotest CLI
        run: npm install -g @vscjava/vscode-autotest

      - name: Download VSIX files
        if: ${{ inputs.vsix_urls != '' }}
        shell: pwsh
        run: |
          New-Item -ItemType Directory -Path vsix -Force | Out-Null
          $urls = "${{ inputs.vsix_urls }}" -split "," | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" }
          foreach ($url in $urls) {
            $fileName = [System.IO.Path]::GetFileName(($url -split '\?')[0])
            Write-Host "Downloading: $url → vsix/$fileName"
            Invoke-WebRequest -Uri $url -OutFile "vsix/$fileName" -UseBasicParsing
          }

      - name: Run ${{ inputs.test_plan }}
        shell: pwsh
        env:
          AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }}
        run: |
          $autotestArgs = @("run", "test-plans/${{ inputs.test_plan }}")
          if (Test-Path vsix) {
            $vsixFiles = (Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { $_.FullName }) -join ","
            if ($vsixFiles) { $autotestArgs += @("--vsix", $vsixFiles) }
          }
          Write-Host "Running: autotest $($autotestArgs -join ' ')"
          & autotest @autotestArgs

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-test-results
          path: test-results/
          retention-days: 30

  # ── Job 3: Aggregate analysis ───────────────────────────
  analyze:
    if: ${{ always() && inputs.test_plan == '' }}
    needs: e2e-test
    runs-on: ubuntu-latest

    steps:
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Install autotest CLI
        run: npm install -g @vscjava/vscode-autotest

      - name: Download all results
        uses: actions/download-artifact@v4
        with:
          pattern: results-*
          path: all-results
          merge-multiple: false

      - name: Organize results
        run: |
          mkdir -p test-results
          for dir in all-results/results-*/; do
            find "$dir" -name "results.json" -exec dirname {} \; | while read d; do
              name=$(basename "$d")
              cp -r "$d" "test-results/$name"
            done
          done
          echo "Organized results:"
          ls test-results/

      - name: Analyze results
        env:
          AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }}
        run: autotest analyze test-results --output test-results

      - name: Write Job Summary
        if: always()
        run: |
          if [ -f test-results/summary.md ]; then
            cat test-results/summary.md >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Upload aggregate results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-aggregate-summary
          path: test-results/summary.md
          retention-days: 30