Flushes the kinesis stream to s3 before deleting resources

Author: lorengordon

.github/actions/test/action.yml

@@ -67,6 +67,45 @@ runs:
         fi
         echo "ARTIFACT_LOCATION=${ARTIFACT_LOCATION}"
 
+        # Flush Firehose streams so all data is written to S3
+        FIREHOSE_STREAMS=$(terraform output -json firehose_delivery_stream_names 2>/dev/null | jq -r '.[] // empty' || true)
+
+        if [ -n "$FIREHOSE_STREAMS" ]; then
+          echo "Flushing Firehose streams..."
+
+          for stream_name in $FIREHOSE_STREAMS; do
+            echo "Reducing buffering interval for stream: $stream_name"
+
+            # Get current delivery stream config and version
+            STREAM_DESC=$(aws firehose describe-delivery-stream --delivery-stream-name "$stream_name" 2>/dev/null || true)
+
+            if [ -n "$STREAM_DESC" ]; then
+              VERSION_ID=$(echo "$STREAM_DESC" | jq -r '.DeliveryStreamDescription.VersionId')
+              DEST_ID=$(echo "$STREAM_DESC" | jq -r '.DeliveryStreamDescription.Destinations[0].DestinationId')
+
+              # Update with reduced time interval (60s) to flush quickly while keeping max size (128 MiB)
+              # Since destination type is not changing, only BufferingHints need to be specified; other config is merged
+              aws firehose update-destination \
+                --delivery-stream-name "$stream_name" \
+                --current-delivery-stream-version-id "$VERSION_ID" \
+                --destination-id "$DEST_ID" \
+                --extended-s3-destination-update '{
+                  "BufferingHints": {
+                    "SizeInMBs": 128,
+                    "IntervalInSeconds": 60
+                  }
+                }' 2>/dev/null || echo "Warning: Could not update stream $stream_name"
+            fi
+          done
+
+          echo "Waiting 60 seconds for Firehose to flush buffered data..."
+          for i in 60 50 40 30 20 10; do
+            echo "  $i seconds remaining..."
+            sleep 10
+          done
+          echo "  Flush complete."
+        fi
+
         if [ "${TF_DESTROY_AFTER_TEST}" = "true" ]; then
           # destroy resources
           terraform destroy -no-color -input=false -auto-approve

README.md

@@ -72,6 +72,9 @@ aws_instance.source_build["centos8stream"]: Still creating... [1m10s elapsed]
 | Name | Type |
 |------|------|
 | [aws_ami.amis](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_iam_instance_profile.builds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_instance_profile) | data source |
+| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
 | [aws_subnet.tfi](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source |
 | [aws_vpc.tfi](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source |
 | [http_http.ip](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source |
@@ -120,6 +123,7 @@ aws_instance.source_build["centos8stream"]: Still creating... [1m10s elapsed]
 | <a name="output_build_id"></a> [build\_id](#output\_build\_id) | n/a |
 | <a name="output_build_slug"></a> [build\_slug](#output\_build\_slug) | n/a |
 | <a name="output_builders"></a> [builders](#output\_builders) | n/a |
+| <a name="output_firehose_delivery_stream_names"></a> [firehose\_delivery\_stream\_names](#output\_firehose\_delivery\_stream\_names) | Kinesis Firehose delivery stream names for flushing before destroy |
 | <a name="output_private_key"></a> [private\_key](#output\_private\_key) | n/a |
 | <a name="output_public_key"></a> [public\_key](#output\_public\_key) | n/a |
 | <a name="output_source_builds"></a> [source\_builds](#output\_source\_builds) | n/a |

outputs.tf

@@ -43,3 +43,10 @@ output "public_key" {
 output "build_slug" {
   value = local.build_slug
 }
+
+output "firehose_delivery_stream_names" {
+  description = "Kinesis Firehose delivery stream names for flushing before destroy"
+  value = {
+    for key, stream in aws_kinesis_firehose_delivery_stream.userdata_logs : key => stream.name
+  }
+}