connect-integration-tests.yml

name: Connect Integration Tests

# This workflow tests Connect extensions against compatible versions of Posit Connect.
# It determines compatibility based on each extension's minimumConnectVersion property,
# which is required in all extension manifests.
#
# Key workflow features:
# 1. Determines all available Connect versions from integration/Makefile
# 2. Reads each extension's minimumConnectVersion from its manifest
# 3. Creates a dynamic test matrix that only includes compatible combinations:
#    - All extensions are ALWAYS tested with the preview version
#    - Extensions are ONLY tested with Connect versions that meet or exceed their minimumConnectVersion
# 4. Runs integration tests only for compatible combinations (instead of skipped jobs polluting results)
# 5. Collects and analyzes test results to determine which extensions passed all their tests

on:
  workflow_call:
    inputs:
      extensions:
        description: "JSON array of extension names to test"
        required: true
        type: string
    outputs:
      successful_extensions:
        description: "JSON array of extension names that have passed all tests"
        value: ${{ jobs.collect-results.outputs.successful_extensions }}
    secrets:
      CONNECT_LICENSE:
        required: true

jobs:
  # Determine the Connect versions to test against
  setup-test:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      versions: ${{ steps.versions.outputs.versions }}
    steps:
      - uses: actions/checkout@v4

      - id: versions
        working-directory: ./integration
        # The `jq` command is "output compact, raw input, slurp, split on new lines, and remove the last (empty) element"
        # This results in a JSON array of Connect versions (e.g., ["2025.01.0", "2024.12.0"])
        run: |
          versions=$(make print-versions | jq -c -Rs 'split("\n") | .[:-1]')
          echo "Versions: $versions"
          echo "versions=$versions" >> "$GITHUB_OUTPUT"

  # Determine compatible Connect versions for each extension based on minimumConnectVersion
  setup-extension-versions:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    needs: setup-test
    outputs:
      matrix: ${{ steps.generate_matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@v5
      
      # This step generates a matrix of compatible extension/version combinations
      # by checking each extension's minimumConnectVersion against available Connect versions
      - id: generate_matrix
        name: Generate extension-version compatibility matrix
        run: |
          # Get the list of all Connect versions from the setup-test job
          ALL_VERSIONS=$(echo '${{ needs.setup-test.outputs.versions }}')
          # Get the list of extensions we need to test from workflow inputs
          EXTENSIONS=$(echo '${{ inputs.extensions }}')
          
          echo "::group::Generating compatibility matrix for extensions"
          echo "All available Connect versions: $ALL_VERSIONS"
          echo "Extensions to test: $EXTENSIONS"
          
          # Extract versions list for direct checking
          CONNECT_VERSIONS=$(echo "$ALL_VERSIONS" | jq -r '.[]')
          echo "::endgroup::"
          
          # This array will hold all the valid extension+version combinations
          # Each entry will be a JSON object: {"extension": "ext-name", "connect_version": "ver"}
          matrix_pairs=()
          
          # Process each extension separately to find compatible versions
          for ext in $(echo "$EXTENSIONS" | jq -r '.[]'); do
            echo "::group::Processing extension: $ext"
            
            # Get minimumConnectVersion from the extension's manifest.json
            MANIFEST_FILE="extensions/$ext/manifest.json"
            
            # Extract the minimumConnectVersion (linting ensures this exists)
            MIN_VERSION=$(jq -r '.extension.minimumConnectVersion' "$MANIFEST_FILE")
            echo "Extension $ext requires minimum Connect version: $MIN_VERSION"
            
            # VALIDATION: Check if minimumConnectVersion exists in CONNECT_VERSIONS list
            if ! echo "$CONNECT_VERSIONS" | grep -q "^$MIN_VERSION$"; then
              echo "❌ ERROR: Extension $ext requires Connect version $MIN_VERSION which is not in CONNECT_VERSIONS list!"
              echo "❌ Please add $MIN_VERSION to CONNECT_VERSIONS in integration/Makefile"
              exit 1
            fi
            
            # RULE 1: Always include the preview version for all extensions
            # This ensures we always test with the latest development version
            matrix_pairs+=("{\"extension\":\"$ext\",\"connect_version\":\"connect-preview\"}")
            echo "✅ Always adding connect-preview for $ext"
            
            # Process each Connect version for this extension
            for version in $(echo "$ALL_VERSIONS" | jq -r '.[]'); do
              # Skip the preview version (already added above)
              if [[ "$version" == "connect-preview" ]]; then
                continue
              fi
              
              # RULE 2: Only include Connect versions that meet or exceed minimumConnectVersion
              # Using sort -V for semantic version comparison (standard in GNU sort)
              # The head -n1 returns the lower version after sorting
              # If the current version is the minimum or higher, it's compatible
              if [[ "$(printf '%s\n' "$MIN_VERSION" "$version" | sort -V | head -n1)" == "$MIN_VERSION" || 
                    "$MIN_VERSION" == "$version" ]]; then
                matrix_pairs+=("{\"extension\":\"$ext\",\"connect_version\":\"$version\"}")
                echo "✅ Adding $version for $ext (meets minimum version $MIN_VERSION)"
              else
                # Skip incompatible versions (lower than minimum requirement)
                echo "⏭️ Excluding $version for $ext (below minimum version $MIN_VERSION)"
              fi
            done
            
            echo "::endgroup::"
          done
          
          # Convert the array of JSON strings into a proper JSON array
          # This creates the final matrix to be used by the test job
          if [ ${#matrix_pairs[@]} -eq 0 ]; then
            matrix_json="[]"
          else
            # Use jq with compact output (-c) for GitHub Actions compatibility
            matrix_json=$(printf '%s\n' "${matrix_pairs[@]}" | jq -c -s .)
          fi
          
          # Set the output that will be used by downstream jobs - no indentation
          echo "matrix=${matrix_json}" >> $GITHUB_OUTPUT
          
          # Output matrix for debugging
          echo "::group::Generated compatibility matrix"
          echo "Generated $(echo "$matrix_json" | jq 'length') combinations"
          # Print raw (no indentation, compact format)
          echo "$matrix_json"
          echo "::endgroup::"

  # Run the Connect integration tests for each extension against compatible Connect versions only
  test:
    runs-on: ubuntu-latest
    timeout-minutes: 15 # Max time to run the integration tests
    needs: [setup-test, setup-extension-versions]
    strategy:
      # Do not fail fast so all extensions and Connect versions are processed
      fail-fast: false
      # Use the dynamically generated matrix from setup-extension-versions
      # This will only create jobs for extension/version combinations that are compatible
      # based on each extension's minimumConnectVersion requirement
      matrix:
        include: ${{ fromJson(needs.setup-extension-versions.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v4

      # Run the integration test for this extension/version combination
      - uses: ./.github/actions/connect-integration-test
        id: test
        with:
          extension-name: ${{ matrix.extension }}
          connect-version: ${{ matrix.connect_version }}
          connect-license: ${{ secrets.CONNECT_LICENSE }}

      # Upload the test report XML files as artifacts for use by downstream jobs
      - uses: actions/upload-artifact@v4
        if: always() && steps.test.outcome != 'cancelled' && steps.test.outcome != 'skipped'
        with:
          name: ${{ matrix.extension }}-${{ matrix.connect_version }}-test-report
          path: integration/reports/*.xml
          retention-days: 7

  # Aggregate test results to determine which extensions passed all their tests
  collect-results:
    needs: [test, setup-extension-versions]
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      successful_extensions: ${{ steps.collect.outputs.successful_extensions }}
    if: always()
    steps:
      - uses: actions/download-artifact@v4
        id: download
        with:
          path: artifacts
          pattern: "*-test-report"
          
      # Collect and analyze results from test reports
      - id: collect
        name: Collect and analyze test results
        run: |
          # Validate inputs first
          extensions='${{ inputs.extensions }}'
          matrix='${{ needs.setup-extension-versions.outputs.matrix }}'

          # Check if setup-extension-versions failed (which would be due to version validation)
          if [[ "${{ needs.setup-extension-versions.result }}" == "failure" ]]; then
            echo "⚠️ setup-extension-versions job failed - likely due to version validation"
            echo "⚠️ No extensions passed validation, returning empty result"
            echo "successful_extensions=[]" >> $GITHUB_OUTPUT
            exit 0
          fi

          if [[ -z "$matrix" || -z "$extensions" ]]; then
            echo "❌ Missing required inputs"
            echo "successful_extensions=[]" >> $GITHUB_OUTPUT
            exit 0
          fi
          
          # Debug info
          echo "::group::Debug Inputs"
          echo "Extensions to check: $extensions"
          echo "Test matrix: $matrix"
          echo "::endgroup::"
          
          # Track extensions that passed ALL version tests
          success_list=()
          
          # Process each extension to determine if it passed all its tests
          for ext in $(echo "$extensions" | jq -r '.[]'); do
            all_passed=true
            echo "📦 Checking extension: $ext"
            
            # Get the list of versions that should have been tested for this extension
            # This uses the filtered matrix so we only check versions that were actually tested
            versions_for_extension=$(echo "$matrix" | jq -r --arg ext "$ext" '[.[] | select(.extension == $ext) | .connect_version]')
            echo "Versions for $ext: $versions_for_extension"
            
            # Check each version that was tested for this extension
            for version in $(echo "$versions_for_extension" | jq -r '.[]'); do
              echo "🔎 Checking $ext @ $version"
              report_dir="artifacts/${ext}-${version}-test-report"
              
              # Check if we have test results for this combination
              if [ ! -d "$report_dir" ]; then
                echo "❌ No test report for $ext @ $version"
                all_passed=false
                break
              fi
              
              # Check for test failures/errors in XML report attributes
              failures=$(grep -o 'failures="[0-9]*"' "$report_dir"/*.xml | sed 's/failures="//g' | sed 's/"//g' | awk '{sum+=$1} END {print sum}' || echo "0")
              errors=$(grep -o 'errors="[0-9]*"' "$report_dir"/*.xml | sed 's/errors="//g' | sed 's/"//g' | awk '{sum+=$1} END {print sum}' || echo "0")
              
              # If there were failures or errors, mark this extension as failed
              if [ "$failures" -gt 0 ] || [ "$errors" -gt 0 ]; then
                echo "❌ Found $failures failures, $errors errors in test suite attributes"
                # Extract and show some error details for debugging
                echo "::group::Error details"
                grep -r -A1 "<failure" "$report_dir" | head -10
                grep -r -A1 "<error" "$report_dir" | head -10
                echo "::endgroup::"
                all_passed=false
                break
              else
                echo "✅ Passed: $ext @ $version"
              fi
            done
            
            # Add to success list if all version tests passed
            if [ "$all_passed" = "true" ]; then
              success_list+=("$ext")
              echo "🎉 SUCCESS: $ext passed ALL versions"
            else
              echo "⚠️ FAILED: $ext failed one or more versions"
            fi
          done
          
          # Format output as a valid JSON array
          # Handle the case where no extensions passed
          if [ ${#success_list[@]} -eq 0 ]; then
            successful_extensions="[]"
          else
            successful_extensions=$(jq -n --arg arr "$(IFS=,; echo "${success_list[*]}")" \
              '$arr | split(",") | map(select(length > 0))' -c)
          fi

          echo "successful_extensions=$successful_extensions" >> $GITHUB_OUTPUT