_shared-run-test.yaml

#name: Run assertoor tests on kurtosis with kubernetes backend

on:
  workflow_call:
    inputs:
      id:
        type: string
        description: "The test identifier."
        required: true
      stagger_seconds:
        type: string
        description: "Seconds between staggered job starts. Uses GitHub Actions cache for atomic slot coordination across all parallel jobs (0 to disable)."
        default: "0"
      name:
        type: string
        description: "The test name."
        required: true
      pairs:
        type: string
        description: "The client pairs to use (format: consensus1-execution1,consensus2-execution2,...)."
        required: true
      worker:
        type: string
        description: "The worker to use (default: ubuntu-latest)."
        default: "ubuntu-latest"
      backend:
        type: string
        description: "The backend type to use (docker, kubernetes)."
        default: "docker"
      kubeCluster:
        type: string
        description: "The name of the cluster to run the test on."
        required: true
      kubeStorageClass:
        type: string
        description: "The kubernetes storage class to run the kurtosis engine with."
        required: true
      clients:
        type: string
        description: "The clients config to use (default: clients/latest-dev.yaml)."
        default: "clients/latest-dev.yaml"
      kurtosis:
        type: string
        description: "The kurtosis network config to use (default: kurtosis-config/default.yaml)."
        default: "kurtosis-config/default.yaml"
      kurtosis_version:
        type: string
        description: "The kurtosis version to use."
        default: ""
      kurtosis_branch:
        type: string
        description: "The branch name for the kurtosis ethereum package to use (default: )."
        default: ""
      assertoor_tests:
        type: string
        description: "The list of assertoor test files to run."
        required: true
      assertoor_image:
        type: string
        description: "The assertoor image to use (default: ethpandaops/assertoor:master)."
        default: "ethpandaops/assertoor:master"
      send_notification:
        type: string
        description: "Send discord notification on test failure (default: false)."
        default: "false"
      use_chatgpt:
        type: string
        description: "Use AI (Claude Code via OpenRouter) to debug and analyze test failures (default: false)."
        default: "false"
      s3_bucket:
        type: string
        description: "The S3 bucket to use for the cache."
        default: ""
      s3_endpoint:
        type: string
        description: "The S3 endpoint to use for the cache."
        default: ""
      s3_insecure:
        type: string
        description: "Whether to use insecure S3 connection."
        default: "false"
      s3_path:
        type: string
        description: "The S3 path to use for the cache."
        default: ""
    secrets:
      KUBECONFIG:
        description: 'Kubernetes config'
        required: true
      DISCORD_HOOK:
        description: 'Discord hook'
        required: false
      CHATGPT_KEY:
        description: 'ChatGPT Key'
        required: false
      OPENROUTER_API_KEY:
        description: 'OpenRouter API Key'
        required: false
      S3_ACCESS_KEY:
        description: 'S3 Access Key'
        required: false
      S3_SECRET_KEY:
        description: 'S3 Secret Key'
        required: false

jobs:
  run_test:
    name: "Run ${{ inputs.id }}"
    runs-on: ${{ fromJson(inputs.worker) }}
    timeout-minutes: 1440
    steps:
    - name: Checkout Repository
      uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
      
    - name: "Stagger job start"
      if: ${{ inputs.stagger_seconds != '0' && inputs.stagger_seconds != '' }}
      shell: bash
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        # Claim a stagger slot using git refs via the GitHub REST API.
        # POST /repos/{owner}/{repo}/git/refs is atomic: returns 201 if the ref
        # is new, 422 if it already exists. This gives us a lock-free counter
        # across all parallel jobs regardless of runner type.

        STAGGER_INTERVAL="${{ inputs.stagger_seconds }}"
        API_URL="${GITHUB_API_URL:-https://api.github.com}"
        REPO="${{ github.repository }}"
        SHA="${{ github.sha }}"
        RUN_ID="${{ github.run_id }}"

        claim_slot() {
          local slot=$1
          local ref="refs/stagger/${RUN_ID}/slot-${slot}"

          local http_code
          http_code=$(curl -s -o /tmp/stagger_response.json -w "%{http_code}" \
            --connect-timeout 5 --max-time 10 \
            -X POST \
            -H "Authorization: token ${GITHUB_TOKEN}" \
            -H "Accept: application/vnd.github+json" \
            -H "X-GitHub-Api-Version: 2022-11-28" \
            "${API_URL}/repos/${REPO}/git/refs" \
            -d "{\"ref\":\"${ref}\",\"sha\":\"${SHA}\"}")

          case "$http_code" in
            201) return 0 ;;  # Created - slot claimed
            422) return 1 ;;  # Already exists - try next slot
            *)
              echo "Warning: Unexpected API response $http_code for stagger slot claim, skipping stagger"
              cat /tmp/stagger_response.json 2>/dev/null || true
              exit 0
              ;;
          esac
        }

        # Claim the next available slot
        slot=0
        claimed=false
        while [ $slot -lt 200 ]; do
          if claim_slot $slot; then
            claimed=true
            break
          fi
          slot=$((slot + 1))
        done

        if [ "$claimed" != "true" ]; then
          echo "Warning: Could not claim stagger slot after 200 attempts, proceeding without delay"
          exit 0
        fi

        delay=$((slot * STAGGER_INTERVAL))
        echo "Claimed stagger slot $slot (delay: ${delay}s / $(( delay / 60 ))m $(( delay % 60 ))s)"

        if [ $delay -gt 0 ]; then
          sleep $delay
        fi

        echo "Stagger complete, proceeding with test"

    - name: "Show Inputs"
      shell: bash
      run: |
        echo '${{ toJson(inputs) }}'

    - name: "Install shell dependencies"
      shell: bash
      run: |
        if [ -z "$(which yq)" ]; then
          sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
        fi

        if [ -z "$(which envsubst)" ]; then
          sudo apt-get update
          sudo apt-get install gettext-base
        fi

        if [ -z "$(which gh)" ]; then
          sudo mkdir -p -m 755 /etc/apt/keyrings
          wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
          sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
          sudo apt update
          sudo apt install gh -y
        fi
    - name: "Generate kurtosis config with client pairs: ${{ inputs.pairs }}"
      shell: bash
      id: clients
      run: |
        clients_file="${{ inputs.clients }}"
        kurtosis_file="${{ inputs.kurtosis }}"
        pairs="${{ inputs.pairs }}"

        mkdir -p ./temp
        touch ./temp/participants.yaml

        echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT

        if [ ! -z "$pairs" ]; then
          resolve_image() {
            image="$1"

            if [ ! -z "$(echo "$1" | grep -E '^.*::.*$')" ]; then
              repository=$(echo "$1" | sed 's/^\(.*\)::\(.*\)$/\1/')
              tag_pattern=$(echo "$1" | sed 's/^\(.*\)::\(.*\)$/\2/')

              # Only resolve Docker Hub images (no registry prefix or docker.io prefix)
              # Skip GCR (gcr.io), GHCR (ghcr.io), and other registries
              if [[ "$repository" =~ ^(gcr\.io|ghcr\.io|[a-z0-9\-]+\.io|[a-z0-9\-]+\.[a-z0-9\-]+)/ ]]; then
                echo "Warning: Image pattern '${1}' uses non-Docker-Hub registry. Pattern matching only works with Docker Hub images." >&2
                echo "$1"
                return
              fi

              while read tag; do
                if [ ! -z "$(echo "$tag" | grep -E $tag_pattern)" ]; then
                  image="${repository}:${tag}"
                  break
                fi
              done <<< $(curl --silent "https://hub.docker.com/v2/repositories/${repository}/tags?page_size=1000" | jq -r ".results[].name")
            fi

            echo "$image"
          }

          echo "participants:" >> ./temp/participants.yaml
          touch ./temp/clients_summary.txt

          client_idx=0
          for pair in $(echo $pairs | tr "," "\n")
          do
            client_idx=$(expr $client_idx + 1)
            if [ "$pair" == "random" ]; then
              cl_type="random"
              el_type="random"
            else
              cl_type=$(echo "$pair" | sed 's/^\(.*\)-\(.*\)$/\1/')
              el_type=$(echo "$pair" | sed 's/^\(.*\)-\(.*\)$/\2/')
            fi

            if [[ "$cl_type" == *"/"* ]]; then
              vc_type=$(echo "$cl_type" | sed 's/^\(.*\)\/\(.*\)$/\2/')
              cl_type=$(echo "$cl_type" | sed 's/^\(.*\)\/\(.*\)$/\1/')
            else
              vc_type=""
            fi

            if [ "$cl_type" == "random" ]; then
              cl_type=$(cat $clients_file | yq -r ".consensus | keys" | shuf | tr -d ' -' | head -n 1)
            fi
            if [ "$vc_type" == "random" ]; then
              vc_type=$(cat $clients_file | yq -r ".consensus | keys" | shuf | tr -d ' -' | head -n 1)
            fi
            if [ "$el_type" == "random" ]; then
              el_type=$(cat $clients_file | yq -r ".execution | keys" | shuf | tr -d ' -' | head -n 1)
            fi

            cl_image="$(cat $clients_file | yq ".consensus.${cl_type}.image")"
            if [ "$cl_image" == "null" ]; then
              echo "Unknown CL client type: $cl_type"
              exit 1
            fi
            cl_image=$(resolve_image "$cl_image")

            if [ -z "$vc_type" ]; then
              vc_image=""
            else
              vc_image="$(cat $clients_file | yq ".consensus.${vc_type}.vc_image // .consensus.${vc_type}.image")"
              if [ "$vc_image" == "null" ]; then
                echo "Unknown VC client type: $vc_type"
                exit 1
              fi
              vc_image=$(resolve_image "$vc_image")
            fi

            el_image="$(cat $clients_file | yq ".execution.${el_type}.image")"
            if [ "$el_image" == "null" ]; then
              echo "Unknown EL client type: $el_type"
              exit 1
            fi
            el_image=$(resolve_image "$el_image")

            echo "  - el_type: $el_type" >> ./temp/participants.yaml
            echo "    el_image: $el_image" >> ./temp/participants.yaml
            el_extra_params="$(cat $clients_file | yq ".execution.${el_type}.params")"
            if [ "$el_extra_params" != "null" ]; then
              echo "    el_extra_params: $el_extra_params" >> ./temp/participants.yaml
            fi

            echo "    cl_type: $cl_type" >> ./temp/participants.yaml
            echo "    cl_image: $cl_image" >> ./temp/participants.yaml
            cl_extra_params="$(cat $clients_file | yq ".consensus.${cl_type}.params")"
            if [ "$cl_extra_params" != "null" ]; then
              echo "    cl_extra_params: $cl_extra_params" >> ./temp/participants.yaml
            fi

            if [ ! -z "$vc_type" ]; then
              echo "    vc_type: $vc_type" >> ./temp/participants.yaml
              echo "    vc_image: $vc_image" >> ./temp/participants.yaml
              vc_extra_params="$(cat $clients_file | yq ".consensus.${cl_type}.vc_params")"
              if [ "$vc_extra_params" != "null" ]; then
                echo "    vc_extra_params: $vc_extra_params" >> ./temp/participants.yaml
              fi
            fi

            echo "    count: 1" >> ./temp/participants.yaml

            echo "Client $client_idx"
            echo "CL: $cl_type  ($cl_image)"
            if [ ! -z "$vc_type" ]; then
              echo "VC: $vc_type  ($vc_image)"
            fi
            echo "EL: $el_type  ($el_image)"

            participant="${el_type}-${cl_type}"
            if [ ! -z "$vc_type" ]; then
              participant="${participant}-${vc_type}"
            fi
            echo "pair $client_idx: $participant" >> ./temp/clients_summary.txt
          done
        fi

        export PUBURL="https://raw.githubusercontent.com/${{ github.repository }}/${{ github.sha }}"

        cp ./temp/participants.yaml ./temp/test-network.yaml
        cat $kurtosis_file | envsubst >> ./temp/test-network.yaml

        kurtosis_cfg="$(cat ./temp/test-network.yaml | yq '.assertoor_params.tests=[]')"
        tests_json=$(
        cat <<"EOF"
        ${{ inputs.assertoor_tests }}
        EOF
        )
        test_index=0
        while read assertoorTest; do
          if [[ $assertoorTest != {* ]]; then
            if [[ $assertoorTest != http* ]]; then
              assertoorTest="$PUBURL/$assertoorTest"
            fi
            assertoorTest="\"$assertoorTest\""
          fi
          
          kurtosis_cfg="$(echo "$kurtosis_cfg" | yq ".assertoor_params.tests[$test_index]=$assertoorTest")"
          test_index=$(expr $test_index + 1)
        done <<< $(echo "$tests_json" | jq -rc '.[]')
        echo "$kurtosis_cfg" > ./temp/test-network.yaml

        # Override assertoor image if provided
        assertoor_image="${{ inputs.assertoor_image }}"
        if ! [ -z "$assertoor_image" ]; then
          echo "Overriding assertoor image to: $assertoor_image"
          kurtosis_cfg="$(cat ./temp/test-network.yaml | yq ".assertoor_params.image = \"$assertoor_image\"")"
          echo "$kurtosis_cfg" > ./temp/test-network.yaml
        fi

        echo ""
        cat ./temp/test-network.yaml

    - name: Setup Kurtosis with docker cache
      if: ${{ inputs.backend == 'docker' }}
      continue-on-error: true
      id: cache
      uses: ethpandaops/kurtosis-cache-github-action@c3e93d993cdb7800c4cb945b65224d3470ab72da # v1.0.1
      with:
        kurtosis_version: ${{ inputs.kurtosis_version }}
        warmup_cache: false
        cache_prefix: "kurtosis-docker"
        s3_access_key: ${{ secrets.S3_ACCESS_KEY }}
        s3_secret_key: ${{ secrets.S3_SECRET_KEY }}
        s3_bucket: ${{ inputs.s3_bucket }}
        s3_endpoint: ${{ inputs.s3_endpoint }}
        s3_insecure: ${{ inputs.s3_insecure }}
        s3_path: ${{ inputs.s3_path }}
    
    - name: Run kurtosis testnet
      id: testnet
      uses: ethpandaops/kurtosis-assertoor-github-action@f64942cbc780df731a731ea9f45765b161d2c8df # v1.0.1
      with:
        kurtosis_extra_args: "--non-blocking-tasks --verbosity DETAILED"
        kurtosis_backend: ${{ inputs.backend }}
        kurtosis_version: ${{ steps.cache.outputs.kurtosis_version || inputs.kurtosis_version }}
        kubernetes_config: ${{ secrets.KUBECONFIG }}
        kubernetes_cluster: ${{ inputs.kubeCluster }}
        kubernetes_storage_class: ${{ inputs.kubeStorageClass }}
        ethereum_package_branch: ${{ inputs.kurtosis_branch }}
        ethereum_package_args: "./temp/test-network.yaml"
        enclave_name: "assertoor-${{ github.run_id }}-${{ inputs.id }}"
        await_assertoor_tests: "false"
        enclave_dump: "false"

    - name: Await assertoor test result
      id: test_result
      uses: ethpandaops/assertoor-github-action@2c0747b6d798a3a646d4339c85258940b8fa66ff # v1
      if: ${{ fromJson(steps.testnet.outputs.services).assertoor.http.url != '' }}
      with:
        kurtosis_enclave_name: "assertoor-${{ github.run_id }}-${{ inputs.id }}"
        assertoor_api_url: ${{ fromJson(steps.testnet.outputs.services).assertoor.http.url }}

    - name: Generate enclave dump
      if: ${{ steps.test_result.outputs.result != '' }}
      id: enclave_dump
      shell: bash
      run: |
        enclave_name="assertoor-${{ github.run_id }}-${{ inputs.id }}"
        tempdir="${{ runner.temp }}/$enclave_name"
        mkdir -p $tempdir/dump
        cp ./temp/test-network.yaml $tempdir/dump/kurtosis-params.yaml
        cd $tempdir/dump
        echo "dump_dir=$tempdir/dump" >> $GITHUB_OUTPUT

        echo "dump_artifact=enclave-dump-assertoor-${{ github.run_id }}-${{ inputs.id }}" >> $GITHUB_OUTPUT

        kurtosis enclave dump $enclave_name

    - name: Install Claude Code
      if: ${{ (failure() || steps.test_result.outputs.result == 'failure') && inputs.use_chatgpt != 'false' }}
      shell: bash
      run: |
        if [ -z "$OPENROUTER_API_KEY" ]; then
          echo "OpenRouter API key not available, skipping Claude Code setup"
          exit 0
        fi
        curl -fsSL https://claude.ai/install.sh | bash
        export PATH="$HOME/.local/bin:$PATH"
        claude --version
      env:
        OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}

    - name: Debug failure with Claude Code
      if: ${{ (failure() || steps.test_result.outputs.result == 'failure') && inputs.use_chatgpt != 'false' }}
      id: claude_code
      timeout-minutes: 15
      shell: bash
      run: |
        export PATH="$HOME/.local/bin:$PATH"
        if [ -z "$OPENROUTER_API_KEY" ]; then
          echo "OpenRouter API key not available, skipping Claude Code analysis"
          exit 0
        fi

        test_status=$(
        cat <<"EOF"
        ${{ steps.test_result.outputs.test_overview }}
        EOF
        )
        failed_test_status=$(
        cat <<"EOF"
        ${{ steps.test_result.outputs.failed_test_details }}
        EOF
        )

        enclave_name="assertoor-${{ github.run_id }}-${{ inputs.id }}"

        # Build the prompt
        cat > ./temp/claude_prompt.md << 'PROMPTEOF'
        You are debugging a failed Ethereum devnet test. The testnet was deployed via Kurtosis and is still running with Docker containers that you can inspect.

        ## Available Commands
        - `kurtosis enclave inspect ENCLAVE_PLACEHOLDER` - List all services in the enclave
        - `kurtosis service logs ENCLAVE_PLACEHOLDER <service_name> --all` - Get full logs for a service
        - `docker ps` - List all running containers
        - `docker logs <container_id>` - Get container logs

        ## Instructions
        1. Start by inspecting the enclave to see all running services
        2. Based on the test failure details below, identify which client services are likely causing issues
        3. Fetch and analyze logs from the suspect services (focus on errors, warnings, panics)
        4. Look for connection failures, consensus issues, or other anomalies
        5. Correlate findings across multiple services if needed

        After your investigation, write your final summary to the file `SUMMARY_PATH_PLACEHOLDER`.
        The summary should be a concise technical analysis (under 2000 characters) covering:
        - Which client(s) are failing and the specific errors found
        - Root cause of the test failure
        - Key log excerpts that demonstrate the issue
        - Suggested next steps for resolution

        Assume assertoor (the test framework) is working correctly if there are no obvious panics - focus on ethereum client issues.
        PROMPTEOF

        # Replace placeholders
        summary_path="$(pwd)/temp/claude_code_summary.txt"
        sed -i "s|ENCLAVE_PLACEHOLDER|$enclave_name|g" ./temp/claude_prompt.md
        sed -i "s|SUMMARY_PATH_PLACEHOLDER|$summary_path|g" ./temp/claude_prompt.md

        # Append dynamic context
        {
          echo ""
          echo "## Environment"
          echo "- **Test ID:** ${{ inputs.id }}"
          echo "- **Test Name:** ${{ inputs.name }}"
          echo "- **Kurtosis Enclave:** $enclave_name"
          echo ""
          echo "## Client Pairs"
          echo "(Format: execution-consensus[-validator])"
          cat ./temp/clients_summary.txt
          echo ""
          echo "## Test Overview"
          echo "$test_status"
          echo ""
          echo "## Failed Test Details"
          echo "$failed_test_status"
        } >> ./temp/claude_prompt.md

        echo "=== Claude Code Prompt ==="
        cat ./temp/claude_prompt.md
        echo "=========================="

        # Run Claude Code with OpenRouter
        # Full output (including intermediate reasoning) goes to claude_code_output.txt
        # Claude is instructed to write its final summary to claude_code_summary.txt
        cat ./temp/claude_prompt.md | claude \
          -p \
          --dangerously-skip-permissions \
          --max-turns 30 \
          --output-format text \
          > ./temp/claude_code_output.txt 2>&1 || true

        echo "=== Claude Code Full Output ==="
        cat ./temp/claude_code_output.txt
        echo "================================"

        # Read the summary file that Claude wrote
        if [ -f ./temp/claude_code_summary.txt ]; then
          echo "Claude Code Summary:"
          cat ./temp/claude_code_summary.txt
        else
          echo "Claude Code did not write a summary file, falling back to output"
          cp ./temp/claude_code_output.txt ./temp/claude_code_summary.txt
        fi

        # Copy to enclave dump if available
        if [ -n "${{ steps.enclave_dump.outputs.dump_dir }}" ]; then
          cp ./temp/claude_prompt.md "${{ steps.enclave_dump.outputs.dump_dir }}/claude_prompt.md" || true
          cp ./temp/claude_code_output.txt "${{ steps.enclave_dump.outputs.dump_dir }}/claude_code_output.txt" || true
          cp ./temp/claude_code_summary.txt "${{ steps.enclave_dump.outputs.dump_dir }}/claude_code_summary.txt" || true
        fi

        # Save summary to step output
        echo "summary<<EOF" >> $GITHUB_OUTPUT
        cat ./temp/claude_code_summary.txt >> $GITHUB_OUTPUT
        echo "EOF" >> $GITHUB_OUTPUT

      env:
        ANTHROPIC_BASE_URL: "https://openrouter.ai/api"
        ANTHROPIC_AUTH_TOKEN: ${{ secrets.OPENROUTER_API_KEY }}
        ANTHROPIC_API_KEY: ""
        OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}

    - name: Upload dump artifact
      if: ${{ steps.enclave_dump.outputs.dump_dir != '' }}
      uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
      with:
        name: "enclave-dump-assertoor-${{ github.run_id }}-${{ inputs.id }}"
        path: "${{ steps.enclave_dump.outputs.dump_dir }}"

    - name: Upload Claude Code output artifact
      if: ${{ always() && steps.claude_code.outcome != 'skipped' }}
      uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
      with:
        name: "claude-code-output-${{ github.run_id }}-${{ inputs.id }}"
        path: |
          ./temp/claude_prompt.md
          ./temp/claude_code_output.txt
          ./temp/claude_code_summary.txt

    - name: Generate github summary (on failure)
      if: ${{ (failure() || steps.test_result.outputs.result == 'failure') }}
      shell: bash
      run: |
        echo "# Test Failed: ${{ inputs.id }}" >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
        echo "## Test Details" >> $GITHUB_STEP_SUMMARY
        echo "- **Test ID:** ${{ inputs.id }}" >> $GITHUB_STEP_SUMMARY
        echo "- **Name:** ${{ inputs.name }}" >> $GITHUB_STEP_SUMMARY
        echo "- **Status:** ${{ steps.test_result.outputs.result }}" >> $GITHUB_STEP_SUMMARY

        start_time="${{ steps.clients.outputs.start_time }}"
        end_time="$(date +%s)"
        runtime="$(expr $end_time - $start_time)"
        echo "- **Run Time:** $(printf '%02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60)))" >> $GITHUB_STEP_SUMMARY

        echo "" >> $GITHUB_STEP_SUMMARY
        echo "## Client Configuration" >> $GITHUB_STEP_SUMMARY
        cat ./temp/clients_summary.txt >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY

        echo "## Failed Test" >> $GITHUB_STEP_SUMMARY
        if [ -z "${{ steps.test_result.outputs.result }}" ]; then
          echo "kurtosis testnet failed during startup" >> $GITHUB_STEP_SUMMARY
        else
          test_status=$(
          cat <<"EOF"
        ${{ steps.test_result.outputs.test_overview }}
        EOF
          )
          echo '```' >> $GITHUB_STEP_SUMMARY
          echo "$test_status" | grep "failure" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
        fi

        claude_summary=$(
        cat <<"EOF"
        ${{ steps.claude_code.outputs.summary }}
        EOF
        )
        if ! [ -z "$claude_summary" ]; then
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "## AI Analysis (Claude Code)" >> $GITHUB_STEP_SUMMARY
          echo "$claude_summary" >> $GITHUB_STEP_SUMMARY
        fi

        echo "" >> $GITHUB_STEP_SUMMARY
        echo "See job logs and artifacts for more details about this failure." >> $GITHUB_STEP_SUMMARY

    - name: Send discord notification (on failure)
      if: ${{ (failure() || steps.test_result.outputs.result == 'failure') && inputs.send_notification == 'true' }}
      shell: bash
      run: |
        if [ -z "$DISCORD_HOOK" ]; then
          echo "discord hook missing"
          exit 1
        fi

        touch ./temp/notification_text.txt
        echo "**Test ID:** ${{ inputs.id }}" >> ./temp/notification_text.txt
        echo "**Name:** ${{ inputs.name }}" >> ./temp/notification_text.txt
        echo "**Status:** ${{ steps.test_result.outputs.result }}" >> ./temp/notification_text.txt

        start_time="${{ steps.clients.outputs.start_time }}"
        end_time="$(date +%s)"
        runtime="$(expr $end_time - $start_time)"
        echo "**Run Time:** $(printf '%02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60)))" >> ./temp/notification_text.txt

        echo "**Clients:**" >> ./temp/notification_text.txt
        cat ./temp/clients_summary.txt >> ./temp/notification_text.txt
        echo "" >> ./temp/notification_text.txt

        echo "**Failed Test:**" >> ./temp/notification_text.txt
        if [ -z "${{ steps.test_result.outputs.result }}" ]; then
          echo "kurtosis testnet failed during startup" >> ./temp/notification_text.txt
        else
        test_status=$(
        cat <<"EOF"
        ${{ steps.test_result.outputs.test_overview }}
        EOF
        )
          echo '```ansi'  >> ./temp/notification_text.txt
          echo "$test_status" | grep "failure" >> ./temp/notification_text.txt
          echo '```'  >> ./temp/notification_text.txt
        fi

        claude_summary=$(
        cat <<"EOF"
        ${{ steps.claude_code.outputs.summary }}
        EOF
        )
        if ! [ -z "$claude_summary" ]; then
          echo "**AI Analysis (Claude Code):**" >> ./temp/notification_text.txt
          echo "$claude_summary" >> ./temp/notification_text.txt
        fi

        echo "Resolve job_id..."
        job_id=$(gh run view ${{ github.run_id }} --json jobs --jq '.jobs[] | select(.name=="${{ github.job }}") | .id')

        if [ -z "$job_id" ]; then
          echo "⚠️ Job ID not found, using default run URL"
          job_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
        else
          job_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${job_id}"
        fi

        echo "Generate discord message..."
        msgjson="{}"
        msgjson=$(echo "$msgjson" | jq -c ".title|=$(echo -n "Assertoor Test Failed: ${{ inputs.id }}" | jq -R -s '.')")
        msgjson=$(echo "$msgjson" | jq -c ".color|=15083560")
        msgjson=$(echo "$msgjson" | jq -c ".url|=$(echo -n "$job_url" | jq -R -s '.')")
        msgjson=$(echo "$msgjson" | jq -c ".description|=$(cat ./temp/notification_text.txt | jq -R -s '.')")

        hookjson="{}"
        hookjson=$(echo "$hookjson" | jq -c ".username|=$(echo -n "Assertoor Test Notification" | jq -R -s '.')")
        hookjson=$(echo "$hookjson" | jq -c ".avatar_url|=$(echo -n "https://raw.githubusercontent.com/ethpandaops/assertoor/master/.github/resources/assertoor.png" | jq -R -s '.')")
        hookjson=$(echo "$hookjson" | jq -c ".embeds|=[${msgjson}]")
        echo $hookjson | jq

        echo "Send discord message..."
        curl -X POST -H 'Content-Type: application/json' -d "$hookjson" $DISCORD_HOOK
      env:
        DISCORD_HOOK: ${{ secrets.DISCORD_HOOK }}
        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

    - name: Return test result
      shell: bash
      run: |
        test_result="${{ steps.test_result.outputs.result }}"
        test_status=$(
        cat <<"EOF"
        ${{ steps.test_result.outputs.test_overview }}
        EOF
        )
        failed_test_status=$(
        cat <<"EOF"
        ${{ steps.test_result.outputs.failed_test_details }}
        EOF
        )
        claude_summary=$(
        cat <<"EOF"
        ${{ steps.claude_code.outputs.summary }}
        EOF
        )

        summary_json="{}"
        summary_json=$(echo "$summary_json" | jq -c --arg value "$test_result" '.test_result = $value')
        summary_json=$(echo "$summary_json" | jq -c --arg value "$test_status" '.test_status = $value')
        summary_json=$(echo "$summary_json" | jq -c --arg value "$failed_test_status" '.failed_test_status = $value')
        summary_json=$(echo "$summary_json" | jq -c --arg value "$claude_summary" '.ai_summary = $value')
        summary_json=$(echo "$summary_json" | jq -c --arg value "${{ steps.enclave_dump.outputs.dump_artifact }}" '.enclave_dump_artifact = $value')
        echo "$summary_json" > ./temp/summary.json

        echo "Test Result:  $test_result"
        echo "$test_status"

        if ! [ "$test_result" == "success" ]; then
          echo ""
          echo "Failed Test Task Status:"
          echo "$failed_test_status"

          if ! [ -z "$claude_summary" ]; then
            echo ""
            echo "AI Analysis (Claude Code):"
            echo "$claude_summary"
          fi

          echo ""
          echo "See 'Await assertoor test result' task or run artifact for detailed logs about this failure!"
          echo ""

          exit 1 # fail action
        fi
    - name: Upload summary artifact
      uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
      if: ${{ always() }}
      with:
        name: "summary-${{ inputs.id }}"
        path: "./temp/summary.json"