From ce9f496e78cffcc02ceb81205a41a63193118a81 Mon Sep 17 00:00:00 2001
From: keeyanghoreshi-rgb <keeyanghoreshi@navapbc.com>
Date: Mon, 4 May 2026 16:41:04 -0400
Subject: [PATCH 1/6] diagnostic

---
 .github/workflows/sonarqube.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml
index 0d33338f7a..03afb0ca7e 100644
--- a/.github/workflows/sonarqube.yml
+++ b/.github/workflows/sonarqube.yml
@@ -67,7 +67,9 @@ jobs:
         with:
           aws-region: ${{ vars.AWS_REGION }}
           role-to-assume: ${{ vars.AB2D_DEV_ROLE }}
-
+      - name: Diagnostic
+        run: |
+          which jq && jq --version
       - name: SonarQube analysis (with coverage)
         run: |
             mvn -ntp -s settings.xml ${RUNNER_DEBUG:+"--debug"} -Dusername=${ARTIFACTORY_USER} -Dpassword=${ARTIFACTORY_PASSWORD} -Drepository_url=${ARTIFACTORY_URL} -Dcheckstyle.skip clean install -pl "!e2e-bfd-test,!e2e-test"  \

From d023deadc28c37adf623bf8cb2824e917932f023 Mon Sep 17 00:00:00 2001
From: keeyanghoreshi-rgb <keeyanghoreshi@navapbc.com>
Date: Tue, 5 May 2026 11:16:42 -0400
Subject: [PATCH 2/6] add wrapper script

---
 .../actions/setup-tofu-logging/action.yml     | 14 +++
 .github/workflows/build-deploy.yml            |  1 +
 .github/workflows/deploy-lambda.yml           |  1 +
 .github/workflows/deploy.yml                  |  1 +
 .github/workflows/microservices-deploy.yml    |  1 +
 .github/workflows/sonarqube.yml               |  3 -
 .github/workflows/tofu-apply.yml              |  1 +
 .github/workflows/tofu-plan.yml               |  1 +
 ops/scripts/tofu-wrapper                      | 87 +++++++++++++++++++
 ops/services/10-core/main.tf                  |  5 ++
 10 files changed, 112 insertions(+), 3 deletions(-)
 create mode 100644 .github/workflows/actions/setup-tofu-logging/action.yml
 create mode 100755 ops/scripts/tofu-wrapper

diff --git a/.github/workflows/actions/setup-tofu-logging/action.yml b/.github/workflows/actions/setup-tofu-logging/action.yml
new file mode 100644
index 0000000000..53f8d5046b
--- /dev/null
+++ b/.github/workflows/actions/setup-tofu-logging/action.yml
@@ -0,0 +1,14 @@
+name: Setup tofu logging wrapper
+description: Adds the tofu wrapper to PATH and sets up a symlink
+
+runs:
+  using: composite
+  steps:
+    - name: Add tofu wrapper to PATH
+      shell: bash
+      run: |
+        echo "TOFU_REAL_BIN=$(which tofu)" >> $GITHUB_ENV
+        chmod +x "$GITHUB_WORKSPACE/ops/scripts/tofu-wrapper
+        mkdir -p $HOME/.local/bin
+        ln -s "$GITHUB_WORKSPACE/ops/scripts/tofu-wrapper" "$HOME/.local/bin/tofu"
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml
index b17a4e6bd7..f09079480f 100644
--- a/.github/workflows/build-deploy.yml
+++ b/.github/workflows/build-deploy.yml
@@ -167,6 +167,7 @@ jobs:
 
       - name: Install tofu
         uses: cmsgov/cdap/actions/setup-tenv@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
+      - uses: ./.github/actions/setup-tofu-logging
 
       - name: Set module-specific variables
         id: module_vars
diff --git a/.github/workflows/deploy-lambda.yml b/.github/workflows/deploy-lambda.yml
index 249d9abc46..0486966a85 100644
--- a/.github/workflows/deploy-lambda.yml
+++ b/.github/workflows/deploy-lambda.yml
@@ -57,6 +57,7 @@ jobs:
   
       - name: Install tofu
         uses: cmsgov/cdap/actions/setup-tenv@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
+      - uses: ./.github/actions/setup-tofu-logging
 
       - name: Tofu plan and apply
         working-directory: ops/services/30-lambda/
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 8aacda08d3..fce2c90041 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -69,6 +69,7 @@ jobs:
 
       - name: Checkout Code
         uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-tofu-logging
 
       - name: Tofu plan and apply - Worker
         working-directory: ops/services/30-worker/
diff --git a/.github/workflows/microservices-deploy.yml b/.github/workflows/microservices-deploy.yml
index aac0ea9e9a..992843fc1f 100644
--- a/.github/workflows/microservices-deploy.yml
+++ b/.github/workflows/microservices-deploy.yml
@@ -96,6 +96,7 @@ jobs:
     
       - name: Install tofu
         uses: cmsgov/cdap/actions/setup-tenv@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
+      - uses: ./.github/actions/setup-tofu-logging
 
       - name: Tofu init & plan
         if: ${{ inputs.service == 'both' || inputs.service == matrix.service }}
diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml
index 03afb0ca7e..fe3f8e3160 100644
--- a/.github/workflows/sonarqube.yml
+++ b/.github/workflows/sonarqube.yml
@@ -67,9 +67,6 @@ jobs:
         with:
           aws-region: ${{ vars.AWS_REGION }}
           role-to-assume: ${{ vars.AB2D_DEV_ROLE }}
-      - name: Diagnostic
-        run: |
-          which jq && jq --version
       - name: SonarQube analysis (with coverage)
         run: |
             mvn -ntp -s settings.xml ${RUNNER_DEBUG:+"--debug"} -Dusername=${ARTIFACTORY_USER} -Dpassword=${ARTIFACTORY_PASSWORD} -Drepository_url=${ARTIFACTORY_URL} -Dcheckstyle.skip clean install -pl "!e2e-bfd-test,!e2e-test"  \
diff --git a/.github/workflows/tofu-apply.yml b/.github/workflows/tofu-apply.yml
index f30746d278..b304898eba 100644
--- a/.github/workflows/tofu-apply.yml
+++ b/.github/workflows/tofu-apply.yml
@@ -83,6 +83,7 @@ jobs:
       - uses: cmsgov/cdap/actions/setup-sops@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
       - uses: cmsgov/cdap/actions/setup-yq@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
       - uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0
+      - uses: ./.github/actions/setup-tofu-logging
         with:
           role-to-assume: arn:aws:iam::${{ contains(fromJSON('["dev", "test"]'), inputs.environment) && secrets.NON_PROD_ACCOUNT || secrets.PROD_ACCOUNT }}:role/delegatedadmin/developer/ab2d-${{ inputs.environment }}-github-actions
           aws-region: ${{ vars.AWS_REGION }}
diff --git a/.github/workflows/tofu-plan.yml b/.github/workflows/tofu-plan.yml
index 5f2c3c0921..c9b94eedd9 100644
--- a/.github/workflows/tofu-plan.yml
+++ b/.github/workflows/tofu-plan.yml
@@ -83,6 +83,7 @@ jobs:
       - uses: cmsgov/cdap/actions/setup-sops@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
       - uses: cmsgov/cdap/actions/setup-yq@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
       - uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0
+      - uses: ./.github/actions/setup-tofu-logging
         with:
           role-to-assume: arn:aws:iam::${{ contains(fromJSON('["dev", "test"]'), inputs.environment) && secrets.NON_PROD_ACCOUNT || secrets.PROD_ACCOUNT }}:role/delegatedadmin/developer/ab2d-${{ inputs.environment }}-github-actions
           aws-region: ${{ vars.AWS_REGION }}
diff --git a/ops/scripts/tofu-wrapper b/ops/scripts/tofu-wrapper
new file mode 100755
index 0000000000..ce987b52b2
--- /dev/null
+++ b/ops/scripts/tofu-wrapper
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# The goal of this script is to run tofu commands
+# while trying to log the output of the command to cloudwatch
+# without interfering with the tofu command/output
+
+set -euo pipefail # force failure on error
+
+LOG_GROUP="${TOFU_LOG_GROUP:-/ab2d/tofu/executions}"
+REGION="${AWS_REGION:-us-east-1}"
+
+TOFU_ENV="${AB2D_ENV:-${ENV:-unknown}}" # dev/test/sbx/prod
+SERVICE="$(basename "$PWD")" # 30-api, 30-worker, etc
+RUN_ID="${GITHUB_RUN_ID:-local-$$}"
+SUBCOMMAND="${1:-unknown}" # plan/apply/init
+LOG_STREAM="$(date -u +%Y/%m/%d)/${TOFU_ENV}/${SERVICE}/${RUN_ID}/${SUBCOMMAND}" # full path
+
+TEMP_FILE="$(mktemp)"
+COMBINED="$(mktemp)"
+trap 'rm -f "$TEMP_FILE" "$COMBINED"' EXIT # clean up temp files on exit
+
+# in order for the wrapper to run, it needs to know where to call tofu.
+# locally, you would just run this script directly.
+# but in the workflows, we use a symlink so that calling "tofu" instead
+# calls this script.
+# Since the "tofu" command now maps to this script, trying to call "tofu" inside this
+# script would just resolve to this script again, causing recursion.
+# Instead, the workflows add the real path to tofu to an environment variable,
+# and then we hijack any calls to tofu by adding to the path our symlinked shortcut
+# so that the shell will find/resolve it first.
+# the result: Calling tofu resolves to this script, and this script can call the real
+# tofu while logging
+REAL_TOFU="${TOFU_REAL_BIN:-tofu}"
+
+START_MS=$(( $(date +%s) * 1000 ))
+
+# don't break on error. If tofu errors, we want to log it.
+set +e
+"$REAL_TOFU" "$@" 2>&1 | tee "$TEMP_FILE"
+exit_code="${PIPESTATUS[0]}"
+set -e # stop allowing errors
+
+END_MS=$(( $(date +%s) * 1000 ))
+DURATION=$(( (END_MS - START_MS) / 1000 ))
+
+_ship_to_cloudwatch() {
+  aws logs create-log-group \
+    --log-group-name "$LOG_GROUP" \
+    --region "$REGION" 2>/dev/null || true
+
+  aws logs put-retention-policy \
+    --log-group-name "$LOG_GROUP" \
+    --region "$REGION" \
+    --retention-in-days 90 2>/dev/null || true
+
+  aws logs create-log-stream \
+    --log-group-name "$LOG_GROUP" \
+    --log-stream-name "$LOG_STREAM" \
+    --region "$REGION" 2>/dev/null || true
+
+  echo "-- tofu $* | env=$TOFU_ENV service=$SERVICE run=$RUN_ID --" > "$COMBINED"
+
+  # strip ANSI codes from the log file
+  sed $'s/\e\\[[0-9;:]*[a-zA-Z]//g' "$TEMP_FILE" >> "$COMBINED"
+
+  echo "-- exit_code=$exit_code duration=${DURATION}s --" >> "$COMBINED"
+
+  # This command processes the output (in file $COMBINED) by splitting it on newline
+  # then converts each line to a pair of index/value with to_entries
+  # then maps each of those pairs into json with two fields:
+  # "timestamp" which is the START_MS plus 1ms per line (just so it prints sequentially)
+  # and "message" which is the content of the line. Empty lines get replaced with a space.
+  EVENTS=$(jq -Rs --argjson ts "$START_MS" \
+    'split("\n") | to_entries | map({timestamp: ($ts + .key), message: (if .value == "" then " " else .value end)})' \
+    "$COMBINED")
+
+  aws logs put-log-events \
+    --log-group-name "$LOG_GROUP" \
+    --log-stream-name "$LOG_STREAM" \
+    --region "$REGION" \
+    --log-events "$EVENTS"
+}
+
+if ! _ship_to_cloudwatch "$@"; then
+  echo "[tofu-wrapper] WARNING: Cloudwatch logging failed" >&2
+fi
+
+exit "$exit_code"
\ No newline at end of file
diff --git a/ops/services/10-core/main.tf b/ops/services/10-core/main.tf
index 39308135ee..02a3f65040 100644
--- a/ops/services/10-core/main.tf
+++ b/ops/services/10-core/main.tf
@@ -369,6 +369,11 @@ resource "aws_vpc_security_group_ingress_rule" "idr_endpoint_https" {
   ip_protocol                  = "tcp"
 }
 
+resource "aws_cloudwatch_log_group" "tofu_executions" {
+  name                = "/ab2d/tofu/executions"
+  retention_in_days   = 90
+  tags                = local.default_tags
+}
 module "idr_db_importer_bucket" {
   source = "github.com/CMSgov/cdap//terraform/modules/bucket?ref=7c070cd2e8c6b1407961c35976553446df8fafd3"
 

From cbf4e9a5fd03c3f7b1f465186a6a4b6f1b388c2e Mon Sep 17 00:00:00 2001
From: keeyanghoreshi-rgb <keeyanghoreshi@navapbc.com>
Date: Tue, 5 May 2026 11:35:05 -0400
Subject: [PATCH 3/6] fix path

---
 .github/workflows/build-deploy.yml         | 2 +-
 .github/workflows/deploy-lambda.yml        | 2 +-
 .github/workflows/deploy.yml               | 2 +-
 .github/workflows/microservices-deploy.yml | 2 +-
 .github/workflows/tofu-apply.yml           | 2 +-
 .github/workflows/tofu-plan.yml            | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml
index f09079480f..2f65cd5734 100644
--- a/.github/workflows/build-deploy.yml
+++ b/.github/workflows/build-deploy.yml
@@ -167,7 +167,7 @@ jobs:
 
       - name: Install tofu
         uses: cmsgov/cdap/actions/setup-tenv@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
-      - uses: ./.github/actions/setup-tofu-logging
+      - uses: ./.github/workflows/actions/setup-tofu-logging
 
       - name: Set module-specific variables
         id: module_vars
diff --git a/.github/workflows/deploy-lambda.yml b/.github/workflows/deploy-lambda.yml
index 0486966a85..948c52ddff 100644
--- a/.github/workflows/deploy-lambda.yml
+++ b/.github/workflows/deploy-lambda.yml
@@ -57,7 +57,7 @@ jobs:
   
       - name: Install tofu
         uses: cmsgov/cdap/actions/setup-tenv@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
-      - uses: ./.github/actions/setup-tofu-logging
+      - uses: ./.github/workflows/actions/setup-tofu-logging
 
       - name: Tofu plan and apply
         working-directory: ops/services/30-lambda/
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index fce2c90041..bcb3520b57 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -69,7 +69,7 @@ jobs:
 
       - name: Checkout Code
         uses: actions/checkout@v4
-      - uses: ./.github/actions/setup-tofu-logging
+      - uses: ./.github/workflows/actions/setup-tofu-logging
 
       - name: Tofu plan and apply - Worker
         working-directory: ops/services/30-worker/
diff --git a/.github/workflows/microservices-deploy.yml b/.github/workflows/microservices-deploy.yml
index 992843fc1f..c764e6f1b5 100644
--- a/.github/workflows/microservices-deploy.yml
+++ b/.github/workflows/microservices-deploy.yml
@@ -96,7 +96,7 @@ jobs:
     
       - name: Install tofu
         uses: cmsgov/cdap/actions/setup-tenv@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
-      - uses: ./.github/actions/setup-tofu-logging
+      - uses: ./.github/workflows/actions/setup-tofu-logging
 
       - name: Tofu init & plan
         if: ${{ inputs.service == 'both' || inputs.service == matrix.service }}
diff --git a/.github/workflows/tofu-apply.yml b/.github/workflows/tofu-apply.yml
index b304898eba..fe6fed48c0 100644
--- a/.github/workflows/tofu-apply.yml
+++ b/.github/workflows/tofu-apply.yml
@@ -83,7 +83,7 @@ jobs:
       - uses: cmsgov/cdap/actions/setup-sops@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
       - uses: cmsgov/cdap/actions/setup-yq@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
       - uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0
-      - uses: ./.github/actions/setup-tofu-logging
+      - uses: ./.github/workflows/actions/setup-tofu-logging
         with:
           role-to-assume: arn:aws:iam::${{ contains(fromJSON('["dev", "test"]'), inputs.environment) && secrets.NON_PROD_ACCOUNT || secrets.PROD_ACCOUNT }}:role/delegatedadmin/developer/ab2d-${{ inputs.environment }}-github-actions
           aws-region: ${{ vars.AWS_REGION }}
diff --git a/.github/workflows/tofu-plan.yml b/.github/workflows/tofu-plan.yml
index c9b94eedd9..30e8adcfdf 100644
--- a/.github/workflows/tofu-plan.yml
+++ b/.github/workflows/tofu-plan.yml
@@ -83,7 +83,7 @@ jobs:
       - uses: cmsgov/cdap/actions/setup-sops@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
       - uses: cmsgov/cdap/actions/setup-yq@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
       - uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0
-      - uses: ./.github/actions/setup-tofu-logging
+      - uses: ./.github/workflows/actions/setup-tofu-logging
         with:
           role-to-assume: arn:aws:iam::${{ contains(fromJSON('["dev", "test"]'), inputs.environment) && secrets.NON_PROD_ACCOUNT || secrets.PROD_ACCOUNT }}:role/delegatedadmin/developer/ab2d-${{ inputs.environment }}-github-actions
           aws-region: ${{ vars.AWS_REGION }}

From 89c3cba5d27b916083e27cc6b71f45e33130ff45 Mon Sep 17 00:00:00 2001
From: keeyanghoreshi-rgb <keeyanghoreshi@navapbc.com>
Date: Tue, 5 May 2026 11:49:36 -0400
Subject: [PATCH 4/6] fix quotes

---
 .github/workflows/actions/setup-tofu-logging/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/actions/setup-tofu-logging/action.yml b/.github/workflows/actions/setup-tofu-logging/action.yml
index 53f8d5046b..8c61464253 100644
--- a/.github/workflows/actions/setup-tofu-logging/action.yml
+++ b/.github/workflows/actions/setup-tofu-logging/action.yml
@@ -8,7 +8,7 @@ runs:
       shell: bash
       run: |
         echo "TOFU_REAL_BIN=$(which tofu)" >> $GITHUB_ENV
-        chmod +x "$GITHUB_WORKSPACE/ops/scripts/tofu-wrapper
+        chmod +x "$GITHUB_WORKSPACE/ops/scripts/tofu-wrapper"
         mkdir -p $HOME/.local/bin
         ln -s "$GITHUB_WORKSPACE/ops/scripts/tofu-wrapper" "$HOME/.local/bin/tofu"
         echo "$HOME/.local/bin" >> $GITHUB_PATH

From 48dc287ff348b03ab36d5de8a8e5df039a17fad9 Mon Sep 17 00:00:00 2001
From: keeyanghoreshi-rgb <keeyanghoreshi@navapbc.com>
Date: Tue, 5 May 2026 12:07:16 -0400
Subject: [PATCH 5/6] try consolidating logs

---
 ops/scripts/tofu-wrapper | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ops/scripts/tofu-wrapper b/ops/scripts/tofu-wrapper
index ce987b52b2..ab101ed413 100755
--- a/ops/scripts/tofu-wrapper
+++ b/ops/scripts/tofu-wrapper
@@ -11,8 +11,7 @@ REGION="${AWS_REGION:-us-east-1}"
 TOFU_ENV="${AB2D_ENV:-${ENV:-unknown}}" # dev/test/sbx/prod
 SERVICE="$(basename "$PWD")" # 30-api, 30-worker, etc
 RUN_ID="${GITHUB_RUN_ID:-local-$$}"
-SUBCOMMAND="${1:-unknown}" # plan/apply/init
-LOG_STREAM="$(date -u +%Y/%m/%d)/${TOFU_ENV}/${SERVICE}/${RUN_ID}/${SUBCOMMAND}" # full path
+LOG_STREAM="$(date -u +%Y/%m/%d)/${TOFU_ENV}/${SERVICE}/${RUN_ID}" # full path
 
 TEMP_FILE="$(mktemp)"
 COMBINED="$(mktemp)"

From 6421c48ec12cb461a835d5f69393e1b6ec62ac6a Mon Sep 17 00:00:00 2001
From: keeyanghoreshi-rgb <keeyanghoreshi@navapbc.com>
Date: Tue, 5 May 2026 12:39:25 -0400
Subject: [PATCH 6/6] preserve log indentation

---
 ops/scripts/tofu-wrapper | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/ops/scripts/tofu-wrapper b/ops/scripts/tofu-wrapper
index ab101ed413..a549865cb1 100755
--- a/ops/scripts/tofu-wrapper
+++ b/ops/scripts/tofu-wrapper
@@ -56,21 +56,14 @@ _ship_to_cloudwatch() {
     --log-stream-name "$LOG_STREAM" \
     --region "$REGION" 2>/dev/null || true
 
-  echo "-- tofu $* | env=$TOFU_ENV service=$SERVICE run=$RUN_ID --" > "$COMBINED"
+  echo "=== tofu $* | env=$TOFU_ENV service=$SERVICE run=$RUN_ID ===" > "$COMBINED"
 
   # strip ANSI codes from the log file
   sed $'s/\e\\[[0-9;:]*[a-zA-Z]//g' "$TEMP_FILE" >> "$COMBINED"
 
-  echo "-- exit_code=$exit_code duration=${DURATION}s --" >> "$COMBINED"
+  echo "=== exit_code=$exit_code duration=${DURATION}s ===" >> "$COMBINED"
 
-  # This command processes the output (in file $COMBINED) by splitting it on newline
-  # then converts each line to a pair of index/value with to_entries
-  # then maps each of those pairs into json with two fields:
-  # "timestamp" which is the START_MS plus 1ms per line (just so it prints sequentially)
-  # and "message" which is the content of the line. Empty lines get replaced with a space.
-  EVENTS=$(jq -Rs --argjson ts "$START_MS" \
-    'split("\n") | to_entries | map({timestamp: ($ts + .key), message: (if .value == "" then " " else .value end)})' \
-    "$COMBINED")
+  EVENTS=$(jq -Rs --argjson ts "$START_MS" '[{timestamp: $ts, message: .}]' "$COMBINED")
 
   aws logs put-log-events \
     --log-group-name "$LOG_GROUP" \