diff --git a/.github/workflows/actions/setup-tofu-logging/action.yml b/.github/workflows/actions/setup-tofu-logging/action.yml new file mode 100644 index 0000000000..8c61464253 --- /dev/null +++ b/.github/workflows/actions/setup-tofu-logging/action.yml @@ -0,0 +1,14 @@ +name: Setup tofu logging wrapper +description: Adds the tofu wrapper to PATH and sets up a symlink + +runs: + using: composite + steps: + - name: Add tofu wrapper to PATH + shell: bash + run: | + echo "TOFU_REAL_BIN=$(which tofu)" >> $GITHUB_ENV + chmod +x "$GITHUB_WORKSPACE/ops/scripts/tofu-wrapper" + mkdir -p $HOME/.local/bin + ln -s "$GITHUB_WORKSPACE/ops/scripts/tofu-wrapper" "$HOME/.local/bin/tofu" + echo "$HOME/.local/bin" >> $GITHUB_PATH diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml index b17a4e6bd7..2f65cd5734 100644 --- a/.github/workflows/build-deploy.yml +++ b/.github/workflows/build-deploy.yml @@ -167,6 +167,7 @@ jobs: - name: Install tofu uses: cmsgov/cdap/actions/setup-tenv@f4c14d47cc20e7f6de9112d7155af1213c9bca5a + - uses: ./.github/workflows/actions/setup-tofu-logging - name: Set module-specific variables id: module_vars diff --git a/.github/workflows/deploy-lambda.yml b/.github/workflows/deploy-lambda.yml index 249d9abc46..948c52ddff 100644 --- a/.github/workflows/deploy-lambda.yml +++ b/.github/workflows/deploy-lambda.yml @@ -57,6 +57,7 @@ jobs: - name: Install tofu uses: cmsgov/cdap/actions/setup-tenv@f4c14d47cc20e7f6de9112d7155af1213c9bca5a + - uses: ./.github/workflows/actions/setup-tofu-logging - name: Tofu plan and apply working-directory: ops/services/30-lambda/ diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8aacda08d3..bcb3520b57 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -69,6 +69,7 @@ jobs: - name: Checkout Code uses: actions/checkout@v4 + - uses: ./.github/workflows/actions/setup-tofu-logging - name: Tofu plan and apply - Worker working-directory: ops/services/30-worker/ diff --git a/.github/workflows/microservices-deploy.yml b/.github/workflows/microservices-deploy.yml index aac0ea9e9a..c764e6f1b5 100644 --- a/.github/workflows/microservices-deploy.yml +++ b/.github/workflows/microservices-deploy.yml @@ -96,6 +96,7 @@ jobs: - name: Install tofu uses: cmsgov/cdap/actions/setup-tenv@f4c14d47cc20e7f6de9112d7155af1213c9bca5a + - uses: ./.github/workflows/actions/setup-tofu-logging - name: Tofu init & plan if: ${{ inputs.service == 'both' || inputs.service == matrix.service }} diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml index 0d33338f7a..fe3f8e3160 100644 --- a/.github/workflows/sonarqube.yml +++ b/.github/workflows/sonarqube.yml @@ -67,7 +67,6 @@ jobs: with: aws-region: ${{ vars.AWS_REGION }} role-to-assume: ${{ vars.AB2D_DEV_ROLE }} - - name: SonarQube analysis (with coverage) run: | mvn -ntp -s settings.xml ${RUNNER_DEBUG:+"--debug"} -Dusername=${ARTIFACTORY_USER} -Dpassword=${ARTIFACTORY_PASSWORD} -Drepository_url=${ARTIFACTORY_URL} -Dcheckstyle.skip clean install -pl "!e2e-bfd-test,!e2e-test" \ diff --git a/.github/workflows/tofu-apply.yml b/.github/workflows/tofu-apply.yml index f30746d278..fe6fed48c0 100644 --- a/.github/workflows/tofu-apply.yml +++ b/.github/workflows/tofu-apply.yml @@ -83,6 +83,7 @@ jobs: - uses: cmsgov/cdap/actions/setup-sops@f4c14d47cc20e7f6de9112d7155af1213c9bca5a - uses: cmsgov/cdap/actions/setup-yq@f4c14d47cc20e7f6de9112d7155af1213c9bca5a - uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0 + - uses: ./.github/workflows/actions/setup-tofu-logging with: role-to-assume: arn:aws:iam::${{ contains(fromJSON('["dev", "test"]'), inputs.environment) && secrets.NON_PROD_ACCOUNT || secrets.PROD_ACCOUNT }}:role/delegatedadmin/developer/ab2d-${{ inputs.environment }}-github-actions aws-region: ${{ vars.AWS_REGION }} diff --git a/.github/workflows/tofu-plan.yml b/.github/workflows/tofu-plan.yml index 5f2c3c0921..30e8adcfdf 100644 --- a/.github/workflows/tofu-plan.yml +++ b/.github/workflows/tofu-plan.yml @@ -83,6 +83,7 @@ jobs: - uses: cmsgov/cdap/actions/setup-sops@f4c14d47cc20e7f6de9112d7155af1213c9bca5a - uses: cmsgov/cdap/actions/setup-yq@f4c14d47cc20e7f6de9112d7155af1213c9bca5a - uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0 + - uses: ./.github/workflows/actions/setup-tofu-logging with: role-to-assume: arn:aws:iam::${{ contains(fromJSON('["dev", "test"]'), inputs.environment) && secrets.NON_PROD_ACCOUNT || secrets.PROD_ACCOUNT }}:role/delegatedadmin/developer/ab2d-${{ inputs.environment }}-github-actions aws-region: ${{ vars.AWS_REGION }} diff --git a/ops/scripts/tofu-wrapper b/ops/scripts/tofu-wrapper new file mode 100755 index 0000000000..a549865cb1 --- /dev/null +++ b/ops/scripts/tofu-wrapper @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# The goal of this script is to run tofu commands +# while trying to log the output of the command to cloudwatch +# without interfering with the tofu command/output + +set -euo pipefail # force failure on error + +LOG_GROUP="${TOFU_LOG_GROUP:-/ab2d/tofu/executions}" +REGION="${AWS_REGION:-us-east-1}" + +TOFU_ENV="${AB2D_ENV:-${ENV:-unknown}}" # dev/test/sbx/prod +SERVICE="$(basename "$PWD")" # 30-api, 30-worker, etc +RUN_ID="${GITHUB_RUN_ID:-local-$$}" +LOG_STREAM="$(date -u +%Y/%m/%d)/${TOFU_ENV}/${SERVICE}/${RUN_ID}" # full path + +TEMP_FILE="$(mktemp)" +COMBINED="$(mktemp)" +trap 'rm -f "$TEMP_FILE" "$COMBINED"' EXIT # clean up temp files on exit + +# in order for the wrapper to run, it needs to know where to call tofu. +# locally, you would just run this script directly. +# but in the workflows, we use a symlink so that calling "tofu" instead +# calls this script. +# Since the "tofu" command now maps to this script, trying to call "tofu" inside this +# script would just resolve to this script again, causing recursion. +# Instead, the workflows add the real path to tofu to an environment variable, +# and then we hijack any calls to tofu by adding to the path our symlinked shortcut +# so that the shell will find/resolve it first. +# the result: Calling tofu resolves to this script, and this script can call the real +# tofu while logging +REAL_TOFU="${TOFU_REAL_BIN:-tofu}" + +START_MS=$(( $(date +%s) * 1000 )) + +# don't break on error. If tofu errors, we want to log it. +set +e +"$REAL_TOFU" "$@" 2>&1 | tee "$TEMP_FILE" +exit_code="${PIPESTATUS[0]}" +set -e # stop allowing errors + +END_MS=$(( $(date +%s) * 1000 )) +DURATION=$(( (END_MS - START_MS) / 1000 )) + +_ship_to_cloudwatch() { + aws logs create-log-group \ + --log-group-name "$LOG_GROUP" \ + --region "$REGION" 2>/dev/null || true + + aws logs put-retention-policy \ + --log-group-name "$LOG_GROUP" \ + --region "$REGION" \ + --retention-in-days 90 2>/dev/null || true + + aws logs create-log-stream \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "$LOG_STREAM" \ + --region "$REGION" 2>/dev/null || true + + echo "=== tofu $* | env=$TOFU_ENV service=$SERVICE run=$RUN_ID ===" > "$COMBINED" + + # strip ANSI codes from the log file + sed $'s/\e\\[[0-9;:]*[a-zA-Z]//g' "$TEMP_FILE" >> "$COMBINED" + + echo "=== exit_code=$exit_code duration=${DURATION}s ===" >> "$COMBINED" + + EVENTS=$(jq -Rs --argjson ts "$START_MS" '[{timestamp: $ts, message: .}]' "$COMBINED") + + aws logs put-log-events \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "$LOG_STREAM" \ + --region "$REGION" \ + --log-events "$EVENTS" +} + +if ! _ship_to_cloudwatch "$@"; then + echo "[tofu-wrapper] WARNING: Cloudwatch logging failed" >&2 +fi + +exit "$exit_code" \ No newline at end of file diff --git a/ops/services/10-core/main.tf b/ops/services/10-core/main.tf index 39308135ee..02a3f65040 100644 --- a/ops/services/10-core/main.tf +++ b/ops/services/10-core/main.tf @@ -369,6 +369,11 @@ resource "aws_vpc_security_group_ingress_rule" "idr_endpoint_https" { ip_protocol = "tcp" } +resource "aws_cloudwatch_log_group" "tofu_executions" { + name = "/ab2d/tofu/executions" + retention_in_days = 90 + tags = local.default_tags +} module "idr_db_importer_bucket" { source = "github.com/CMSgov/cdap//terraform/modules/bucket?ref=7c070cd2e8c6b1407961c35976553446df8fafd3"