From 2dbaedb5d2f17d1495270ec9afb2cae1e610fd3a Mon Sep 17 00:00:00 2001 From: Matthias Kay Date: Thu, 18 Sep 2025 10:21:51 +0200 Subject: [PATCH 1/7] refactor: rename lambda function file and update references to it --- .../lambda/{lambda_function.py => terminate_runners.py} | 0 modules/terminate-agent-hook/main.tf | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename modules/terminate-agent-hook/lambda/{lambda_function.py => terminate_runners.py} (100%) diff --git a/modules/terminate-agent-hook/lambda/lambda_function.py b/modules/terminate-agent-hook/lambda/terminate_runners.py similarity index 100% rename from modules/terminate-agent-hook/lambda/lambda_function.py rename to modules/terminate-agent-hook/lambda/terminate_runners.py diff --git a/modules/terminate-agent-hook/main.tf b/modules/terminate-agent-hook/main.tf index c8076c45a..d52a40aca 100644 --- a/modules/terminate-agent-hook/main.tf +++ b/modules/terminate-agent-hook/main.tf @@ -5,12 +5,12 @@ # terminating orphaned runner instances. # ---------------------------------------------------------------------------- locals { - source_sha256 = filesha256("${path.module}/lambda/lambda_function.py") + source_sha256 = filesha256("${path.module}/lambda/terminate_runners.py") } data "archive_file" "terminate_runner_instances_lambda" { type = "zip" - source_file = "${path.module}/lambda/lambda_function.py" + source_file = "${path.module}/lambda/terminate_runners.py" output_path = "builds/lambda_function_${local.source_sha256}.zip" output_file_mode = "0666" } @@ -52,7 +52,7 @@ resource "aws_lambda_function" "terminate_runner_instances" { # checkov:skip=CKV_AWS_116:We should think about having a dead letter queue for this lambda # checkov:skip=CKV_AWS_272:Code signing would be a nice enhancement, but I guess we can live without it here architectures = ["arm64"] - description = "Lifecycle hook for terminating GitLab runner agent instances" + description = "Lifecycle hook for terminating GitLab runner instances" filename = data.archive_file.terminate_runner_instances_lambda.output_path source_code_hash = data.archive_file.terminate_runner_instances_lambda.output_base64sha256 function_name = "${var.environment}-${var.name}" From 97d0faef6405f58de006a866197471efa3d6ddc6 Mon Sep 17 00:00:00 2001 From: Matthias Kay Date: Thu, 18 Sep 2025 10:24:28 +0200 Subject: [PATCH 2/7] fix: update handler call to use empty dict for event parameter --- modules/terminate-agent-hook/lambda/terminate_runners.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/terminate-agent-hook/lambda/terminate_runners.py b/modules/terminate-agent-hook/lambda/terminate_runners.py index 23936f90c..7ca875660 100644 --- a/modules/terminate-agent-hook/lambda/terminate_runners.py +++ b/modules/terminate-agent-hook/lambda/terminate_runners.py @@ -13,7 +13,6 @@ from botocore.exceptions import ClientError import json import os -import sys def ec2_list(client, **args): @@ -273,4 +272,4 @@ def handler(event, context): if __name__ == "__main__": - handler(None, None) + handler({}, None) From 09abcfe46345fbe2def4a2b702f6ad9998db83f5 Mon Sep 17 00:00:00 2001 From: Matthias Kay Date: Thu, 18 Sep 2025 10:47:30 +0200 Subject: [PATCH 3/7] fix: update EC2 instance termination logic to include stopping state and force termination --- modules/terminate-agent-hook/lambda/requirements.txt | 4 ++-- modules/terminate-agent-hook/lambda/terminate_runners.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/terminate-agent-hook/lambda/requirements.txt b/modules/terminate-agent-hook/lambda/requirements.txt index aaf8daa50..e3b1647c9 100644 --- a/modules/terminate-agent-hook/lambda/requirements.txt +++ b/modules/terminate-agent-hook/lambda/requirements.txt @@ -1,2 +1,2 @@ -boto3 ==1.40.30 -botocore ==1.40.30 +boto3 == 1.40.30 +botocore == 1.40.30 diff --git a/modules/terminate-agent-hook/lambda/terminate_runners.py b/modules/terminate-agent-hook/lambda/terminate_runners.py index 7ca875660..e214d654d 100644 --- a/modules/terminate-agent-hook/lambda/terminate_runners.py +++ b/modules/terminate-agent-hook/lambda/terminate_runners.py @@ -11,6 +11,7 @@ """ import boto3 from botocore.exceptions import ClientError +from types_boto3_ec2.client import EC2Client import json import os @@ -32,7 +33,7 @@ def ec2_list(client, **args): ec2_instances = client.describe_instances(Filters=[ { "Name": "instance-state-name", - "Values": ['running', 'pending'], + "Values": ['running', 'pending', 'stopping', 'stopped'], }, { "Name": "tag:gitlab-runner-parent-id", @@ -247,7 +248,7 @@ def handler(event, context): "Message": f"Terminating instances {', '.join(_terminate_list)}" })) try: - client.terminate_instances(InstanceIds=_terminate_list, DryRun=False) + client.terminate_instances(InstanceIds=_terminate_list, DryRun=False, Force=True) print(json.dumps({ "Level": "info", From c55020cdb942cfbd62ea364661369b780a4ab19f Mon Sep 17 00:00:00 2001 From: Matthias Kay Date: Thu, 18 Sep 2025 10:59:14 +0200 Subject: [PATCH 4/7] feat: implement retry mechanism for terminating instances --- .../lambda/terminate_runners.py | 48 ++++++++++++++----- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/modules/terminate-agent-hook/lambda/terminate_runners.py b/modules/terminate-agent-hook/lambda/terminate_runners.py index e214d654d..39ce277bd 100644 --- a/modules/terminate-agent-hook/lambda/terminate_runners.py +++ b/modules/terminate-agent-hook/lambda/terminate_runners.py @@ -9,6 +9,8 @@ This is rudimentary and doesn't check if a build runner has a current job. """ +import time + import boto3 from botocore.exceptions import ClientError from types_boto3_ec2.client import EC2Client @@ -222,6 +224,33 @@ def remove_unused_ssh_key_pairs(client, executor_name_part): "Exception": str(error) })) +def retry(func, max_retries=3, retry_delay=2): + """ + Generic retry helper for functions that may raise exceptions. + :param func: Function to execute + :param max_retries: Maximum number of attempts + :param retry_delay: Delay between attempts in seconds + :return: Result of func if successful, None otherwise + """ + for attempt in range(1, max_retries + 1): + try: + return func() + except Exception as ex: + print(json.dumps({ + "Level": "exception", + "Attempt": attempt, + "Exception": str(ex) + })) + + if attempt < max_retries: + time.sleep(retry_delay) + else: + print(json.dumps({ + "Level": "error", + "Message": f"Failed after {max_retries} retries" + })) + + return None # context not used: this is the interface for a AWS Lambda function defined by AWS # pylint: disable=unused-argument @@ -247,20 +276,13 @@ def handler(event, context): "Level": "info", "Message": f"Terminating instances {', '.join(_terminate_list)}" })) - try: - client.terminate_instances(InstanceIds=_terminate_list, DryRun=False, Force=True) - print(json.dumps({ - "Level": "info", - "Message": "Instances terminated" - })) - # catch everything here and log it - # pylint: disable=broad-exception-caught - except Exception as ex: - print(json.dumps({ - "Level": "exception", - "Exception": str(ex) - })) + retry(lambda: client.terminate_instances(InstanceIds=_terminate_list, DryRun=False, Force=True)) + + print(json.dumps({ + "Level": "info", + "Message": "Instances terminated" + })) else: print(json.dumps({ "Level": "info", From 65336234b7f96cf08483a6dbdefe8e3708ef6818 Mon Sep 17 00:00:00 2001 From: Matthias Kay Date: Thu, 18 Sep 2025 11:00:16 +0200 Subject: [PATCH 5/7] fix: correct retry loop to include the maximum number of retries --- modules/terminate-agent-hook/lambda/terminate_runners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/terminate-agent-hook/lambda/terminate_runners.py b/modules/terminate-agent-hook/lambda/terminate_runners.py index 39ce277bd..4cd392b48 100644 --- a/modules/terminate-agent-hook/lambda/terminate_runners.py +++ b/modules/terminate-agent-hook/lambda/terminate_runners.py @@ -232,7 +232,7 @@ def retry(func, max_retries=3, retry_delay=2): :param retry_delay: Delay between attempts in seconds :return: Result of func if successful, None otherwise """ - for attempt in range(1, max_retries + 1): + for attempt in range(max_retries + 1): try: return func() except Exception as ex: From 36717fc328b9bf744ae60395e73dc2f5b0192fee Mon Sep 17 00:00:00 2001 From: Matthias Kay Date: Thu, 18 Sep 2025 11:11:29 +0200 Subject: [PATCH 6/7] refactor: remove unused EC2Client import from terminate_runners.py --- modules/terminate-agent-hook/lambda/terminate_runners.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/terminate-agent-hook/lambda/terminate_runners.py b/modules/terminate-agent-hook/lambda/terminate_runners.py index 4cd392b48..2866a6bcd 100644 --- a/modules/terminate-agent-hook/lambda/terminate_runners.py +++ b/modules/terminate-agent-hook/lambda/terminate_runners.py @@ -13,7 +13,6 @@ import boto3 from botocore.exceptions import ClientError -from types_boto3_ec2.client import EC2Client import json import os From e321af5823c4fce23c24119318afeef249ed3d8b Mon Sep 17 00:00:00 2001 From: Matthias Kay Date: Thu, 18 Sep 2025 11:23:23 +0200 Subject: [PATCH 7/7] 1298 --- modules/terminate-agent-hook/lambda/terminate_runners.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/terminate-agent-hook/lambda/terminate_runners.py b/modules/terminate-agent-hook/lambda/terminate_runners.py index 2866a6bcd..677e8c22a 100644 --- a/modules/terminate-agent-hook/lambda/terminate_runners.py +++ b/modules/terminate-agent-hook/lambda/terminate_runners.py @@ -9,12 +9,12 @@ This is rudimentary and doesn't check if a build runner has a current job. """ +import json +import os import time import boto3 from botocore.exceptions import ClientError -import json -import os def ec2_list(client, **args): @@ -67,7 +67,7 @@ def ec2_list(client, **args): # Handle other instances without a parent that are still running. _other_child = client.describe_instances(InstanceIds=[tag['Value']]) # The specified parent is still in the inventory as 'terminated' - if (len(_other_child['Reservations']) > 0): + if len(_other_child['Reservations']) > 0: if _other_child['Reservations'][0]['Instances'][0]['State']['Name'] == "terminated": _terminate_list.append(instance['InstanceId']) _msg_suffix = "is terminated." @@ -234,6 +234,8 @@ def retry(func, max_retries=3, retry_delay=2): for attempt in range(max_retries + 1): try: return func() + # catch everything here and log it + # pylint: disable=broad-exception-caught except Exception as ex: print(json.dumps({ "Level": "exception",