feat(bench): Updated tx inclusion benchmarking

.github/workflows/bench-inclusion-point.yml

@@ -0,0 +1,62 @@
+name: Bench Inclusion Point
+
+# Reusable: runs ONE inclusion-sweep TPS point against an ALREADY-DEPLOYED
+# network (SKIP_NETWORK_DEPLOY=1). The caller (nightly-bench-inclusion-sweep.yml)
+# deploys the network once and calls this for each point (1/5/10 TPS), chaining
+# the calls so the points run sequentially on the same network. Deploy / wait /
+# teardown are done once by the caller, not here.
+
+on:
+  workflow_call:
+    inputs:
+      tps:
+        description: "Target TPS for this point"
+        required: true
+        type: string
+      namespace:
+        description: "k8s namespace of the already-deployed network"
+        required: true
+        type: string
+      network:
+        description: "Network env name (environments/<network>.env)"
+        required: false
+        type: string
+        default: bench-inclusion-sweep
+      docker_image:
+        description: "Full aztec docker image (already deployed)"
+        required: true
+        type: string
+      source_ref:
+        description: "Git ref to checkout for the bench scripts"
+        required: true
+        type: string
+      sweep_id:
+        description: "Shared sweep id so the dashboard groups the points"
+        required: true
+        type: string
+
+jobs:
+  bench:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
+        with:
+          ref: ${{ inputs.source_ref }}
+      - name: Run ${{ inputs.tps }} TPS inclusion point
+        timeout-minutes: 120
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          RUN_ID: ${{ github.run_id }}
+          AWS_SHUTDOWN_TIME: 180
+          NO_SPOT: 1
+          SKIP_NETWORK_DEPLOY: "1"
+          TARGET_TPS: ${{ inputs.tps }}
+          BENCH_SWEEP_ID: ${{ inputs.sweep_id }}
+          BENCH_SWEEP_LABEL: inclusion-sweep
+        run: ./.github/ci3.sh network-inclusion-sweep ${{ inputs.network }} ${{ inputs.namespace }} "${{ inputs.docker_image }}"

.github/workflows/nightly-bench-10tps.yml → .github/workflows/nightly-bench-inclusion-sweep.yml

@@ -1,8 +1,15 @@
-name: Nightly Bench 10 TPS
+name: Nightly Bench Inclusion Sweep
+
+# Inclusion sweep: deploys ONE network and runs the
+# 1/5/10 TPS points SEQUENTIALLY against it, all tagged with a shared sweepId so
+# the dashboard groups them. The scraper drains the mempool to zero between
+# points. Each point is the reusable bench-inclusion-point.yml workflow
+# (SKIP_NETWORK_DEPLOY=1); the next point's call `needs` the previous one so they
+# don't overlap. Deploy / wait-for-first-block / teardown happen once.
 
 on:
   schedule:
-    - cron: "30 6 * * *"
+    - cron: "30 3 * * *"
   workflow_dispatch:
     inputs:
       docker_image:
@@ -19,7 +26,7 @@ on:
         type: string
 
 concurrency:
-  group: nightly-bench-10tps-${{ github.ref }}
+  group: nightly-bench-inclusion-sweep-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -29,6 +36,7 @@ jobs:
       docker_image: ${{ steps.docker-image.outputs.docker_image }}
       image_label: ${{ steps.docker-image.outputs.image_label }}
       source_ref: ${{ steps.docker-image.outputs.source_ref }}
+      sweep_id: ${{ steps.sweep.outputs.sweep_id }}
     steps:
       - name: Checkout
         uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
@@ -71,130 +79,132 @@ jobs:
           echo "Using source ref: $source_ref"
 
       - name: Verify source git ref
-        id: verify-source
         run: |
           set -euo pipefail
           source_ref="${{ steps.docker-image.outputs.source_ref }}"
           git fetch --depth 1 origin "refs/tags/${source_ref}:refs/tags/${source_ref}"
-          source_sha=$(git rev-parse "${source_ref}^{}")
-          echo "Nightly source commit: $source_sha"
+          git rev-parse "${source_ref}^{}"
 
       - name: Check if Docker image exists
         run: |
           DOCKER_IMAGE="${{ steps.docker-image.outputs.docker_image }}"
-          echo "Checking if Docker image exists: $DOCKER_IMAGE"
           if docker manifest inspect "$DOCKER_IMAGE" > /dev/null 2>&1; then
             echo "Docker image exists: $DOCKER_IMAGE"
           else
             echo "Docker image does not exist: $DOCKER_IMAGE"
             exit 1
           fi
 
-  deploy-bench-10tps-network:
+      - name: Compute shared sweep id
+        id: sweep
+        run: |
+          # Shared across all three points so the dashboard groups the sweep.
+          echo "sweep_id=incl-$(date -u +%Y%m%d)-${{ github.run_id }}" >> "$GITHUB_OUTPUT"
+
+  # ---- Deploy the single network the whole sweep runs against ----
+  deploy:
     needs: select-image
     uses: ./.github/workflows/deploy-network.yml
     with:
-      network: bench-10tps
-      namespace: bench-10tps
+      network: bench-inclusion-sweep
+      namespace: bench-inclusion-sweep
       aztec_docker_image: ${{ needs.select-image.outputs.docker_image }}
       ref: ${{ needs.select-image.outputs.source_ref }}
       notify_on_failure: false
     secrets: inherit
 
-  wait-for-first-l2-block:
-    needs:
-      - select-image
-      - deploy-bench-10tps-network
+  wait:
+    needs: [select-image, deploy]
     runs-on: ubuntu-latest
     timeout-minutes: 120
     steps:
-      - name: Checkout
-        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
+      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
         with:
           ref: ${{ needs.select-image.outputs.source_ref }}
-
-      - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
+      - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
         with:
           credentials_json: ${{ secrets.GCP_SA_KEY }}
-
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1
+      - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1
         with:
           install_components: gke-gcloud-auth-plugin
-
       - name: Wait for first L2 block
         env:
           GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-        run: |
-          cd spartan
-          ./bootstrap.sh wait_for_l2_block bench-10tps
+          NAMESPACE: bench-inclusion-sweep
+        run: cd spartan && ./bootstrap.sh wait_for_l2_block bench-inclusion-sweep
 
-  benchmark:
-    needs:
-      - select-image
-      - wait-for-first-l2-block
+  # ---- Three points, sequential, on the SAME network ----
+  point-1tps:
+    needs: [select-image, wait]
+    uses: ./.github/workflows/bench-inclusion-point.yml
+    with:
+      tps: "1"
+      namespace: bench-inclusion-sweep
+      docker_image: ${{ needs.select-image.outputs.docker_image }}
+      source_ref: ${{ needs.select-image.outputs.source_ref }}
+      sweep_id: ${{ needs.select-image.outputs.sweep_id }}
+    secrets: inherit
+
+  # 5 TPS runs after 1 TPS finishes — gated on the network being up (wait), not
+  # on the 1 TPS result, so a failed point drops only itself.
+  point-5tps:
+    needs: [select-image, wait, point-1tps]
+    if: ${{ !cancelled() && needs.wait.result == 'success' }}
+    uses: ./.github/workflows/bench-inclusion-point.yml
+    with:
+      tps: "5"
+      namespace: bench-inclusion-sweep
+      docker_image: ${{ needs.select-image.outputs.docker_image }}
+      source_ref: ${{ needs.select-image.outputs.source_ref }}
+      sweep_id: ${{ needs.select-image.outputs.sweep_id }}
+    secrets: inherit
+
+  point-10tps:
+    needs: [select-image, wait, point-5tps]
+    if: ${{ !cancelled() && needs.wait.result == 'success' }}
+    uses: ./.github/workflows/bench-inclusion-point.yml
+    with:
+      tps: "10"
+      namespace: bench-inclusion-sweep
+      docker_image: ${{ needs.select-image.outputs.docker_image }}
+      source_ref: ${{ needs.select-image.outputs.source_ref }}
+      sweep_id: ${{ needs.select-image.outputs.sweep_id }}
+    secrets: inherit
+
+  # ---- Tear the network down once, regardless of point outcomes ----
+  cleanup:
+    if: always()
+    needs: [select-image, deploy, wait, point-1tps, point-5tps, point-10tps]
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
+      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
         with:
           ref: ${{ needs.select-image.outputs.source_ref }}
-
-      - name: Run 10 TPS benchmark
-        timeout-minutes: 240
+      - name: Cleanup network resources
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
           BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
           GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
           GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
-          RUN_ID: ${{ github.run_id }}
-          AWS_SHUTDOWN_TIME: 240
           NO_SPOT: 1
-          SKIP_NETWORK_DEPLOY: "1"
-        run: |
-          ./.github/ci3.sh network-bench-10tps bench-10tps bench-10tps "${{ needs.select-image.outputs.docker_image }}"
-
-  cleanup:
-    if: always()
-    needs:
-      - select-image
-      - deploy-bench-10tps-network
-      - wait-for-first-l2-block
-      - benchmark
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
-        with:
-          ref: ${{ needs.select-image.outputs.source_ref }}
-
-      - name: Cleanup network resources
-        uses: ./.github/actions/network-teardown
-        with:
-          env_file: bench-10tps
-          namespace: bench-10tps
-          gcp_sa_key: ${{ secrets.GCP_SA_KEY }}
-          gcp_project_id: ${{ secrets.GCP_PROJECT_ID }}
+        run: ./.github/ci3.sh network-teardown bench-inclusion-sweep bench-inclusion-sweep
 
   notify-failure:
     if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' }}
     needs:
       - select-image
-      - deploy-bench-10tps-network
-      - wait-for-first-l2-block
-      - benchmark
-      - cleanup
+      - deploy
+      - wait
+      - point-1tps
+      - point-5tps
+      - point-10tps
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
+      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
         with:
           ref: ${{ needs.select-image.outputs.source_ref }}
-
       - name: Notify Slack on failure
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
@@ -203,6 +213,6 @@ jobs:
           IMAGE="${{ needs.select-image.outputs.image_label || 'unknown' }}"
           RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           ./ci3/slack_notify_with_claudebox_kickoff "#alerts-next-scenario" \
-            "Nightly 10 TPS benchmark FAILED (image ${IMAGE}): <${RUN_URL}|View Run> (🤖)" \
-            "Nightly 10 TPS benchmark failed (image ${IMAGE}). CI run: ${RUN_URL}. Investigate the benchmark failure and identify the root cause." \
+            "Nightly inclusion sweep FAILED (image ${IMAGE}): <${RUN_URL}|View Run> (🤖)" \
+            "Nightly inclusion sweep failed (image ${IMAGE}). CI run: ${RUN_URL}. Deploy/wait failures fail the whole sweep; a failed individual point drops only that point." \
             --link "$RUN_URL"

bootstrap.sh

@@ -936,6 +936,36 @@ case "$cmd" in
     bench_merge
     cache_upload spartan-bench-10tps-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json
     ;;
+  "ci-network-inclusion-sweep")
+    # Args: <env_file> <namespace> [docker_image]
+    # Runs one inclusion-sweep point (TARGET_TPS) on the given network.
+    # The v4 run JSON (tagged BENCH_SWEEP_ID) is uploaded to GCS inside
+    # bench_inclusion_point; deploy/teardown of each point's namespace is done by
+    # the workflow, so this is normally called with SKIP_NETWORK_DEPLOY=1.
+    export CI=1
+    env_file="${1:?env_file is required}"
+    namespace="${2:?namespace is required}"
+    docker_image="${3:-}"
+    build
+    export NAMESPACE="$namespace"
+    if [ "${SKIP_NETWORK_DEPLOY:-0}" != "1" ]; then
+      # If no docker image provided, build and push to aztecdev
+      if [ -z "$docker_image" ]; then
+        release-image/bootstrap.sh push_pr
+        docker_image="aztecprotocol/aztecdev:$(git rev-parse HEAD)"
+      fi
+      export AZTEC_DOCKER_IMAGE="$docker_image"
+      spartan/bootstrap.sh network_deploy "${env_file}"
+    else
+      echo "SKIP_NETWORK_DEPLOY=1, running inclusion-sweep point (${TARGET_TPS:-10} TPS) against existing network '$namespace'."
+    fi
+    # Run one inclusion-sweep point (TARGET_TPS / BENCH_SWEEP_ID from env).
+    spartan/bootstrap.sh bench_inclusion_point "${env_file}"
+    rm -rf bench-out
+    mkdir -p bench-out
+    bench_merge
+    cache_upload spartan-bench-inclusion-${TARGET_TPS:-10}tps-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json
+    ;;
   "ci-network-teardown")
     # Args: <env_file> <namespace>
     # Tears down a deployed network.

ci.sh

@@ -298,6 +298,21 @@ case "$cmd" in
     [ "${SKIP_NETWORK_DEPLOY:-0}" = "1" ] && skip_network_deploy=1
     bootstrap_ec2 "SKIP_NETWORK_DEPLOY=$skip_network_deploy ./bootstrap.sh ci-network-bench-10tps $*"
     ;;
+  network-inclusion-sweep)
+    # Args: <env_file> <namespace> [docker_image]
+    # Runs one inclusion-sweep point at TARGET_TPS against an existing
+    # network, tagged with BENCH_SWEEP_ID. The workflow deploys/tears down each
+    # point's namespace separately, so this is normally called with
+    # SKIP_NETWORK_DEPLOY=1. TARGET_TPS / BENCH_SWEEP_ID / BENCH_SWEEP_LABEL come
+    # from the caller's env and are threaded into the remote command.
+    export CI_DASHBOARD="network"
+    export JOB_ID="x-${2:?namespace is required}-network-inclusion-sweep" CPUS=16
+    export AWS_SHUTDOWN_TIME=${AWS_SHUTDOWN_TIME:-180}
+    export INSTANCE_POSTFIX="n-incl-sweep"
+    skip_network_deploy=0
+    [ "${SKIP_NETWORK_DEPLOY:-0}" = "1" ] && skip_network_deploy=1
+    bootstrap_ec2 "TARGET_TPS=${TARGET_TPS:-10} BENCH_SWEEP_ID=${BENCH_SWEEP_ID:-} BENCH_SWEEP_LABEL=${BENCH_SWEEP_LABEL:-inclusion-sweep} SKIP_NETWORK_DEPLOY=$skip_network_deploy ./bootstrap.sh ci-network-inclusion-sweep $*"
+    ;;
   network-teardown)
     # Args: <scenario> <namespace>
     export CI_DASHBOARD="network"

spartan/.gitignore

@@ -35,5 +35,6 @@ environments/*
 !environments/alpha-net.env
 !environments/mbps-pipeline.env
 !environments/bench-10tps.env
+!environments/bench-inclusion-sweep.env
 *.tfvars
 !terraform/deploy-external-secrets/*.tfvars