AB2D-7245 tofu logging

.github/workflows/actions/setup-tofu-logging/action.yml

@@ -0,0 +1,14 @@
+name: Setup tofu logging wrapper
+description: Adds the tofu wrapper to PATH and sets up a symlink
+
+runs:
+  using: composite
+  steps:
+    - name: Add tofu wrapper to PATH
+      shell: bash
+      run: |
+        echo "TOFU_REAL_BIN=$(which tofu)" >> $GITHUB_ENV
+        chmod +x "$GITHUB_WORKSPACE/ops/scripts/tofu-wrapper"
+        mkdir -p $HOME/.local/bin
+        ln -s "$GITHUB_WORKSPACE/ops/scripts/tofu-wrapper" "$HOME/.local/bin/tofu"
+        echo "$HOME/.local/bin" >> $GITHUB_PATH

.github/workflows/build-deploy.yml

@@ -167,6 +167,7 @@ jobs:
 
       - name: Install tofu
         uses: cmsgov/cdap/actions/setup-tenv@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
+      - uses: ./.github/workflows/actions/setup-tofu-logging
 
       - name: Set module-specific variables
         id: module_vars

.github/workflows/deploy-lambda.yml

@@ -57,6 +57,7 @@ jobs:
 
       - name: Install tofu
         uses: cmsgov/cdap/actions/setup-tenv@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
+      - uses: ./.github/workflows/actions/setup-tofu-logging
 
       - name: Tofu plan and apply
         working-directory: ops/services/30-lambda/

.github/workflows/deploy.yml

@@ -69,6 +69,7 @@ jobs:
 
       - name: Checkout Code
         uses: actions/checkout@v4
+      - uses: ./.github/workflows/actions/setup-tofu-logging
 
       - name: Tofu plan and apply - Worker
         working-directory: ops/services/30-worker/

.github/workflows/microservices-deploy.yml

@@ -96,6 +96,7 @@ jobs:
 
       - name: Install tofu
         uses: cmsgov/cdap/actions/setup-tenv@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
+      - uses: ./.github/workflows/actions/setup-tofu-logging
 
       - name: Tofu init & plan
         if: ${{ inputs.service == 'both' || inputs.service == matrix.service }}

.github/workflows/sonarqube.yml

@@ -67,7 +67,6 @@ jobs:
         with:
           aws-region: ${{ vars.AWS_REGION }}
           role-to-assume: ${{ vars.AB2D_DEV_ROLE }}
-
       - name: SonarQube analysis (with coverage)
         run: |
             mvn -ntp -s settings.xml ${RUNNER_DEBUG:+"--debug"} -Dusername=${ARTIFACTORY_USER} -Dpassword=${ARTIFACTORY_PASSWORD} -Drepository_url=${ARTIFACTORY_URL} -Dcheckstyle.skip clean install -pl "!e2e-bfd-test,!e2e-test"  \

.github/workflows/tofu-apply.yml

@@ -83,6 +83,7 @@ jobs:
       - uses: cmsgov/cdap/actions/setup-sops@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
       - uses: cmsgov/cdap/actions/setup-yq@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
       - uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0
+      - uses: ./.github/workflows/actions/setup-tofu-logging
         with:
           role-to-assume: arn:aws:iam::${{ contains(fromJSON('["dev", "test"]'), inputs.environment) && secrets.NON_PROD_ACCOUNT || secrets.PROD_ACCOUNT }}:role/delegatedadmin/developer/ab2d-${{ inputs.environment }}-github-actions
           aws-region: ${{ vars.AWS_REGION }}

.github/workflows/tofu-plan.yml

@@ -83,6 +83,7 @@ jobs:
       - uses: cmsgov/cdap/actions/setup-sops@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
       - uses: cmsgov/cdap/actions/setup-yq@f4c14d47cc20e7f6de9112d7155af1213c9bca5a
       - uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0
+      - uses: ./.github/workflows/actions/setup-tofu-logging
         with:
           role-to-assume: arn:aws:iam::${{ contains(fromJSON('["dev", "test"]'), inputs.environment) && secrets.NON_PROD_ACCOUNT || secrets.PROD_ACCOUNT }}:role/delegatedadmin/developer/ab2d-${{ inputs.environment }}-github-actions
           aws-region: ${{ vars.AWS_REGION }}

ops/scripts/tofu-wrapper

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# The goal of this script is to run tofu commands
+# while trying to log the output of the command to cloudwatch
+# without interfering with the tofu command/output
+
+set -euo pipefail # force failure on error
+
+LOG_GROUP="${TOFU_LOG_GROUP:-/ab2d/tofu/executions}"
+REGION="${AWS_REGION:-us-east-1}"
+
+TOFU_ENV="${AB2D_ENV:-${ENV:-unknown}}" # dev/test/sbx/prod
+SERVICE="$(basename "$PWD")" # 30-api, 30-worker, etc
+RUN_ID="${GITHUB_RUN_ID:-local-$$}"
+LOG_STREAM="$(date -u +%Y/%m/%d)/${TOFU_ENV}/${SERVICE}/${RUN_ID}" # full path
+
+TEMP_FILE="$(mktemp)"
+COMBINED="$(mktemp)"
+trap 'rm -f "$TEMP_FILE" "$COMBINED"' EXIT # clean up temp files on exit
+
+# in order for the wrapper to run, it needs to know where to call tofu.
+# locally, you would just run this script directly.
+# but in the workflows, we use a symlink so that calling "tofu" instead
+# calls this script.
+# Since the "tofu" command now maps to this script, trying to call "tofu" inside this
+# script would just resolve to this script again, causing recursion.
+# Instead, the workflows add the real path to tofu to an environment variable,
+# and then we hijack any calls to tofu by adding to the path our symlinked shortcut
+# so that the shell will find/resolve it first.
+# the result: Calling tofu resolves to this script, and this script can call the real
+# tofu while logging
+REAL_TOFU="${TOFU_REAL_BIN:-tofu}"
+
+START_MS=$(( $(date +%s) * 1000 ))
+
+# don't break on error. If tofu errors, we want to log it.
+set +e
+"$REAL_TOFU" "$@" 2>&1 | tee "$TEMP_FILE"
+exit_code="${PIPESTATUS[0]}"
+set -e # stop allowing errors
+
+END_MS=$(( $(date +%s) * 1000 ))
+DURATION=$(( (END_MS - START_MS) / 1000 ))
+
+_ship_to_cloudwatch() {
+  aws logs create-log-group \
+    --log-group-name "$LOG_GROUP" \
+    --region "$REGION" 2>/dev/null || true
+
+  aws logs put-retention-policy \
+    --log-group-name "$LOG_GROUP" \
+    --region "$REGION" \
+    --retention-in-days 90 2>/dev/null || true
+
+  aws logs create-log-stream \
+    --log-group-name "$LOG_GROUP" \
+    --log-stream-name "$LOG_STREAM" \
+    --region "$REGION" 2>/dev/null || true
+
+  echo "=== tofu $* | env=$TOFU_ENV service=$SERVICE run=$RUN_ID ===" > "$COMBINED"
+
+  # strip ANSI codes from the log file
+  sed $'s/\e\\[[0-9;:]*[a-zA-Z]//g' "$TEMP_FILE" >> "$COMBINED"
+
+  echo "=== exit_code=$exit_code duration=${DURATION}s ===" >> "$COMBINED"
+
+  EVENTS=$(jq -Rs --argjson ts "$START_MS" '[{timestamp: $ts, message: .}]' "$COMBINED")
+
+  aws logs put-log-events \
+    --log-group-name "$LOG_GROUP" \
+    --log-stream-name "$LOG_STREAM" \
+    --region "$REGION" \
+    --log-events "$EVENTS"
+}
+
+if ! _ship_to_cloudwatch "$@"; then
+  echo "[tofu-wrapper] WARNING: Cloudwatch logging failed" >&2
+fi
+
+exit "$exit_code"

ops/services/10-core/main.tf

@@ -369,6 +369,11 @@ resource "aws_vpc_security_group_ingress_rule" "idr_endpoint_https" {
   ip_protocol                  = "tcp"
 }
 
+resource "aws_cloudwatch_log_group" "tofu_executions" {
+  name                = "/ab2d/tofu/executions"
+  retention_in_days   = 90
+  tags                = local.default_tags
+}
 module "idr_db_importer_bucket" {
   source = "github.com/CMSgov/cdap//terraform/modules/bucket?ref=7c070cd2e8c6b1407961c35976553446df8fafd3"