agentforce-script-validate.yml

# This workflow is an example used during the Irish Dreamin'
# Agentforce DevOps: state of the art
# By @nabondance

name: Agentforce Script Validate
# This workflow runs Salesforce Agent tests and aggregates the results.
# It is split into several steps for better readability and understanding.

on:
  pull_request:
    types: [opened, synchronize, ready_for_review, reopened]
    branches: [ main ]

jobs:
  setup-deploy:
    runs-on: ubuntu-latest
    container: salesforce/cli:latest-slim

    steps:
      - name: Checkout Code
        uses: actions/checkout@v5

      - name: 'Scope project to agentforce-script'
        run: cp agentforce-script/sfdx-project.json sfdx-project.json

      - name: Install pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 10

      - name: Setup Node.js
        id: setup-node
        uses: actions/setup-node@v4
        with:
          node-version-file: .node-version
          cache: pnpm
          cache-dependency-path: 'pnpm-lock.yaml'

      - name: Install Dependencies
        id: pnpm-install
        run: pnpm install

      - name: 'Authenticate to Dev Hub'
        run: |
         echo "${{ secrets.DEVHUB_SFDX_AUTH_URL }}" > ./authfile
         sf org login sfdx-url --sfdxurlfile=authfile --alias=devhub

      - name: 'Get org from pool'
        run: |
          pnpm sfp pool fetch --targetdevhubusername=devhub --tag=ci-pool --alias=so-ci

      - name: 'Deploy Agentforce Script to org'
        run: |
          sf project deploy start --target-org=so-ci --manifest=agentforce-script/manifest.xml --wait=30

      - name: 'Publish Agentforce authoring bundles'
        run: bash agentforce-script/scripts/publishAgents.sh so-ci

      - name: 'Activate Agentforce agents'
        run: bash agentforce-script/scripts/activateAgents.sh so-ci

      - name: 'Patch test definitions with latest agent version'
        run: bash agentforce-script/scripts/patchTestVersions.sh so-ci

      - name: 'Deploy Agentforce Tests to org'
        run: |
          sf project deploy start --target-org=so-ci --source-dir=agentforce-script/force-app/main/default/aiEvaluationDefinitions --wait=30

      - name: 'Capture org auth URL'
        id: capture-org
        run: |
          AUTH_URL=$(sf org display --target-org=so-ci --verbose --json | tr -d '\000-\037' | jq -r '.result.sfdxAuthUrl')
          mkdir -p auth-artifact
          echo "$AUTH_URL" > auth-artifact/auth-url.txt

      - name: Upload auth URL artifact
        uses: actions/upload-artifact@v4
        with:
          name: org-auth-url
          path: auth-artifact/

  list-tests:
    name: List Agent Tests
    needs: setup-deploy
    runs-on: ubuntu-latest
    container: salesforce/cli:latest-slim
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}

    steps:
      - uses: actions/checkout@v5

      - name: Download auth URL artifact
        uses: actions/download-artifact@v4
        with:
          name: org-auth-url
          path: auth-artifact

      - name: Authenticate to org
        run: |
          AUTH_URL=$(cat auth-artifact/auth-url.txt)
          if [ -z "$AUTH_URL" ] || [ "$AUTH_URL" = "null" ]; then
            echo "ERROR: Auth URL is empty or null from artifact"
            exit 1
          fi
          echo "$AUTH_URL" > ./authfileci
          sf org login sfdx-url --sfdxurlfile=authfileci --alias=so-ci

      - name: List agent tests and set matrix
        id: set-matrix
        run: |
          TESTS=$(sf agent test list --target-org=so-ci --json | jq -c '[.result[].fullName]')
          if [ "$TESTS" == "[]" ]; then
            echo "No tests found. Failing early."
            exit 1
          fi
          echo "matrix={\"test\":$TESTS}" >> "$GITHUB_OUTPUT"

  run-agent-test:
    name: Run Agent Test - ${{ matrix.test }}
    needs: [setup-deploy, list-tests]
    runs-on: ubuntu-latest
    container: salesforce/cli:latest-slim
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.list-tests.outputs.matrix) }}

    steps:
      - uses: actions/checkout@v5

      - name: Download auth URL artifact
        uses: actions/download-artifact@v4
        with:
          name: org-auth-url
          path: auth-artifact

      - name: Authenticate to org
        run: |
          AUTH_URL=$(cat auth-artifact/auth-url.txt)
          echo "$AUTH_URL" > ./authfileci
          sf org login sfdx-url --sfdxurlfile=authfileci --alias=so-ci

      - name: Run test and get result
        id: test
        run: |
          mkdir -p test-results
          RUN_ID=$(sf agent test run --target-org=so-ci --api-name="${{ matrix.test }}" --wait 10 --json | jq -r '.result.runId')
          RESULT=$(sf agent test results --target-org=so-ci --job-id="$RUN_ID" --json)
          echo "$RESULT" | tr -d '\000-\037' > "test-results/${{ matrix.test }}.json"

      - name: Upload individual result
        uses: actions/upload-artifact@v4
        with:
          name: agent-test-results-${{ matrix.test }}
          path: test-results/

  validate-results:
    name: Validate Results
    needs: run-agent-test
    runs-on: ubuntu-latest

    steps:
      - name: Download all test results
        uses: actions/download-artifact@v4
        with:
          path: all-results
          pattern: agent-test-results-*
          merge-multiple: true

      - name: Summarize test outcomes
        id: summary
        run: |
          total=0
          passed=0
          failed=0

          ls -al all-results/

          if ! ls all-results/ 1> /dev/null 2>&1; then
            echo "No test result files found"
            exit 1
          fi

          for file in all-results/*; do
            echo "Processing file: $file"

            if ! jq -e '.result.testCases' "$file" > /dev/null 2>&1; then
              echo "Skipping invalid or malformed JSON file: $file"
              continue
            fi

            p=$(jq '[.result.testCases[]? | .testResults[]? | select(.result == "PASS")] | length' "$file" 2>/dev/null || echo "0")
            f=$(jq '[.result.testCases[]? | .testResults[]? | select(.result == "FAILURE")] | length' "$file" 2>/dev/null || echo "0")

            total=$((total + p + f))
            passed=$((passed + p))
            failed=$((failed + f))

            echo "File $file: $p passed, $f failed"
          done

          if [ $total -eq 0 ]; then
            echo "No test results found in any files"
            exit 1
          fi

          percentage=$((passed * 100 / total))
          echo "TOTAL=$total" >> "$GITHUB_OUTPUT"
          echo "PASSED=$passed" >> "$GITHUB_OUTPUT"
          echo "FAILED=$failed" >> "$GITHUB_OUTPUT"
          echo "PERCENT=$percentage" >> "$GITHUB_OUTPUT"

      - name: Display summary
        run: |
          echo "=================================="
          echo "          Agent Test Summary      "
          echo "=================================="
          echo "Total Tests :   ${{ steps.summary.outputs.TOTAL }}"
          echo "Tests Passed:   ${{ steps.summary.outputs.PASSED }}"
          echo "Tests Failed:   ${{ steps.summary.outputs.FAILED }}"
          echo "Pass        :   ${{ steps.summary.outputs.PERCENT }}%"
          echo "=================================="

      - name: Enforce pass threshold
        if: ${{ steps.summary.outputs.PERCENT < 75 }}
        run: |
          echo "Agent tests failed threshold (75%). Only ${{ steps.summary.outputs.PERCENT }}% passed."
          exit 1

  cleanup:
    name: Return Org to Pool
    needs: [setup-deploy, validate-results]
    if: always()
    runs-on: ubuntu-latest
    container: salesforce/cli:latest-slim

    steps:
      - name: Checkout Code
        uses: actions/checkout@v5

      - name: Install pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 10

      - name: Install Dependencies
        run: pnpm install

      - name: Download auth URL artifact
        uses: actions/download-artifact@v4
        with:
          name: org-auth-url
          path: auth-artifact

      - name: Authenticate to DevHub
        run: |
          echo "${{ secrets.DEVHUB_SFDX_AUTH_URL }}" > ./authfile
          sf org login sfdx-url --sfdxurlfile=authfile --alias=devhub

      - name: Authenticate to org
        run: |
          AUTH_URL=$(cat auth-artifact/auth-url.txt)
          echo "$AUTH_URL" > ./authfileci
          sf org login sfdx-url --sfdxurlfile=authfileci --alias=so-ci

      - name: Return org to pool
        run: |
          ORG_ID=$(sf org display -o so-ci --json | jq -r '.result.id' | cut -c 1-15)
          sf data update record -o devhub --sobject ScratchOrgInfo --where "ScratchOrg='$ORG_ID'" --values "Allocation_status__c='Available'"