E2E AutoTest #20
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E AutoTest | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| test_plan: | |
| description: "Test plan to run (leave empty for all)" | |
| required: false | |
| default: "" | |
| type: string | |
| jobs: | |
| # ── Job 1: Discover test plans ────────────────────────── | |
| discover: | |
| if: ${{ inputs.test_plan == '' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.scan.outputs.matrix }} | |
| steps: | |
| - name: Checkout javaext-autotest | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: wenytang-ms/javaext-autotest | |
| - name: Scan test plans | |
| id: scan | |
| run: | | |
| plans=$(ls test-plans/*.yaml | xargs -I{} basename {} .yaml | grep -v java-fresh-import | jq -R . | jq -sc .) | |
| echo "matrix=$plans" >> "$GITHUB_OUTPUT" | |
| echo "Found plans: $plans" | |
| # ── Job 2: Run each test plan in parallel ─────────────── | |
| e2e-test: | |
| if: ${{ inputs.test_plan == '' }} | |
| needs: discover | |
| runs-on: windows-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| plan: ${{ fromJson(needs.discover.outputs.matrix) }} | |
| steps: | |
| - name: Checkout javaext-autotest | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: wenytang-ms/javaext-autotest | |
| path: javaext-autotest | |
| - name: Checkout vscode-java (test projects) | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: redhat-developer/vscode-java | |
| path: vscode-java | |
| - name: Checkout eclipse.jdt.ls (Gradle test projects) | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: eclipse-jdtls/eclipse.jdt.ls | |
| path: eclipse.jdt.ls | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Setup Java 25 (for java25 test plans) | |
| if: contains(matrix.plan, 'java25') | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 25-ea | |
| - name: Create JDK 25 path | |
| if: contains(matrix.plan, 'java25') | |
| shell: pwsh | |
| run: | | |
| New-Item -ItemType Junction -Path "C:\Program Files\Java\jdk-25" -Target $env:JAVA_HOME | |
| - name: Setup Java 21 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 21 | |
| - name: Install and build autotest | |
| working-directory: javaext-autotest | |
| run: npm ci && npm run build | |
| - name: Run ${{ matrix.plan }} | |
| shell: pwsh | |
| working-directory: javaext-autotest | |
| env: | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} | |
| run: npx autotest run "test-plans/${{ matrix.plan }}.yaml" | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: results-${{ matrix.plan }} | |
| path: javaext-autotest/test-results/ | |
| retention-days: 30 | |
| # ── Job 2b: Run a single test plan (when specified) ───── | |
| e2e-single: | |
| if: ${{ inputs.test_plan != '' }} | |
| runs-on: windows-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout javaext-autotest | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: wenytang-ms/javaext-autotest | |
| path: javaext-autotest | |
| - name: Checkout vscode-java | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: redhat-developer/vscode-java | |
| path: vscode-java | |
| - name: Checkout eclipse.jdt.ls | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: eclipse-jdtls/eclipse.jdt.ls | |
| path: eclipse.jdt.ls | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Setup Java 25 | |
| if: contains(inputs.test_plan, 'java25') | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 25-ea | |
| - name: Create JDK 25 path | |
| if: contains(inputs.test_plan, 'java25') | |
| shell: pwsh | |
| run: | | |
| New-Item -ItemType Junction -Path "C:\Program Files\Java\jdk-25" -Target $env:JAVA_HOME | |
| - name: Setup Java 21 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 21 | |
| - name: Install and build autotest | |
| working-directory: javaext-autotest | |
| run: npm ci && npm run build | |
| - name: Run ${{ inputs.test_plan }} | |
| shell: pwsh | |
| working-directory: javaext-autotest | |
| env: | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} | |
| run: npx autotest run "test-plans/${{ inputs.test_plan }}" | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-test-results | |
| path: javaext-autotest/test-results/ | |
| retention-days: 30 | |
| # ── Job 3: Aggregate analysis ─────────────────────────── | |
| analyze: | |
| if: ${{ always() && inputs.test_plan == '' }} | |
| needs: e2e-test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout javaext-autotest | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: wenytang-ms/javaext-autotest | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Install and build autotest | |
| run: npm ci && npm run build | |
| - name: Download all results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: results-* | |
| path: all-results | |
| merge-multiple: false | |
| - name: Organize results | |
| run: | | |
| mkdir -p test-results | |
| for dir in all-results/results-*/; do | |
| plan=$(basename "$dir" | sed 's/^results-//') | |
| if [ -d "$dir/$plan" ]; then | |
| cp -r "$dir/$plan" "test-results/$plan" | |
| elif [ -d "$dir" ]; then | |
| # Results may be nested differently | |
| find "$dir" -name "results.json" -exec dirname {} \; | while read d; do | |
| name=$(basename "$d") | |
| cp -r "$d" "test-results/$name" | |
| done | |
| fi | |
| done | |
| echo "Organized results:" | |
| ls test-results/ | |
| - name: Analyze results | |
| env: | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} | |
| run: npx autotest analyze test-results --output test-results | |
| - name: Write Job Summary | |
| if: always() | |
| run: | | |
| if [ -f test-results/summary.md ]; then | |
| cat test-results/summary.md >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Upload aggregate results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-aggregate-summary | |
| path: test-results/summary.md | |
| retention-days: 30 |