From 4c7e10147c4353fdf11ee02eb5c23917868ffba7 Mon Sep 17 00:00:00 2001 From: seshubaws <116689586+seshubaws@users.noreply.github.com> Date: Thu, 12 Feb 2026 11:19:41 -0800 Subject: [PATCH 1/6] Update notify-slack.yml to add sleep (#8646) --- .github/workflows/notify-slack.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/notify-slack.yml b/.github/workflows/notify-slack.yml index c87ec70eefe..d42971a4f57 100644 --- a/.github/workflows/notify-slack.yml +++ b/.github/workflows/notify-slack.yml @@ -12,8 +12,11 @@ jobs: permissions: contents: read steps: + - name: Wait for label to be applied + run: sleep 10s + shell: bash - name: Send External PR Notification - if: github.event_name == 'pull_request' && github.event.label.name == 'pr/external' + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'pr/external') uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # v2 env: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} From 463163d941220250359a8cdf8146fe018688d5e4 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Thu, 12 Feb 2026 17:58:26 -0800 Subject: [PATCH 2/6] fix: use tag prefix matching to clean up samcli/lambda-* images (#8647) Docker's images.list(name='samcli/lambda') does exact repository matching and won't match repositories like 'samcli/lambda-python'. This caused stale images to persist across parameterized test classes, leading to flaky test_download_two_layers failures where Layer2 should overwrite Layer1 but the image was never rebuilt. Fix by: 1. Adding _cleanup_samcli_images() that lists all images and filters by 'samcli/lambda-' tag prefix 2. Using this method in both tearDown and tearDownClass 3. Fixing the same pattern in TestLayerVersionThatDoNotCreateCache --- .../local/invoke/test_integrations_cli.py | 47 +++++++++++++------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/tests/integration/local/invoke/test_integrations_cli.py b/tests/integration/local/invoke/test_integrations_cli.py index 2971ab00773..e2b51aa9eb1 100644 --- a/tests/integration/local/invoke/test_integrations_cli.py +++ b/tests/integration/local/invoke/test_integrations_cli.py @@ -803,13 +803,27 @@ class TestLayerVersionBase(InvokeIntegBase): def setUp(self): self.layer_cache = Path().home().joinpath("integ_layer_cache") + @staticmethod + def _cleanup_samcli_images(docker_client): + """Remove all samcli/lambda-* images. + + Docker's images.list(name="samcli/lambda") does exact repository matching + and won't match repositories like "samcli/lambda-python". We list all images + and filter by tag prefix instead. + """ + try: + all_images = docker_client.images.list() + for image in all_images: + for tag in image.tags: + if tag.startswith("samcli/lambda-"): + docker_client.remove_image_safely(image.id, force=True) + break + except Exception: + pass + def tearDown(self): docker_client = get_validated_container_client() - samcli_images = docker_client.images.list(name="samcli/lambda") - for image in samcli_images: - # Use strategy pattern method for runtime-aware image cleanup - docker_client.remove_image_safely(image.id, force=True) - + self._cleanup_samcli_images(docker_client) shutil.rmtree(str(self.layer_cache), ignore_errors=True) @classmethod @@ -823,10 +837,7 @@ def tearDownClass(cls): cls.layer_utils.delete_layers() # Added to handle the case where ^C failed the test due to invalid cleanup of layers docker_client = get_validated_container_client() - samcli_images = docker_client.images.list(name="samcli/lambda") - for image in samcli_images: - # Use strategy pattern method for runtime-aware image cleanup - docker_client.remove_image_safely(image.id, force=True) + cls._cleanup_samcli_images(docker_client) integ_layer_cache_dir = Path().home().joinpath("integ_layer_cache") if integ_layer_cache_dir.exists(): shutil.rmtree(str(integ_layer_cache_dir)) @@ -1069,12 +1080,18 @@ def setUp(self): def tearDown(self): docker_client = get_validated_container_client() - # Use strategy pattern method for runtime-aware image cleanup - # This handles both Docker and Finch cleanup strategies automatically - samcli_images = docker_client.images.list(name="samcli/lambda") - for image in samcli_images: - # Use strategy pattern method that handles runtime-specific cleanup logic - docker_client.remove_image_safely(image.id, force=True) + # Use the same prefix-based cleanup as TestLayerVersionBase. + # Docker's images.list(name="samcli/lambda") does exact repository matching + # and won't match "samcli/lambda-python" etc. + try: + all_images = docker_client.images.list() + for image in all_images: + for tag in image.tags: + if tag.startswith("samcli/lambda-"): + docker_client.remove_image_safely(image.id, force=True) + break + except Exception: + pass def test_layer_does_not_exist(self): self.layer_utils.upsert_layer(LayerUtils.generate_layer_name(), "LayerOneArn", "layer1.zip") From bc739a5b629a168707b5eb3ed354aa68f49c06ba Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Fri, 13 Feb 2026 03:26:07 +0000 Subject: [PATCH 3/6] fix: remove fallback in count_running_containers that causes flaky warm container tests The count_running_containers method had a fallback that returned the count of ALL SAM CLI containers when MODE env var filtering found no matches. This caused AssertionError: 3 != 2 when stale containers from other tests were present. Now it strictly counts only containers matching this test's unique MODE UUID and uses exact string matching. --- .../workflows/test-warm-containers-fix.yml | 43 +++++++++++++++++++ .../local/start_api/test_start_api.py | 15 ++----- .../local/start_lambda/test_start_lambda.py | 15 ++----- 3 files changed, 49 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/test-warm-containers-fix.yml diff --git a/.github/workflows/test-warm-containers-fix.yml b/.github/workflows/test-warm-containers-fix.yml new file mode 100644 index 00000000000..43d96680916 --- /dev/null +++ b/.github/workflows/test-warm-containers-fix.yml @@ -0,0 +1,43 @@ +name: Test Warm Containers Fix + +on: + pull_request: + branches: [develop] + workflow_dispatch: + +permissions: + contents: read + +env: + SAM_CLI_DEV: 1 + SAM_CLI_TELEMETRY: 0 + SAM_CLI_CONTAINER_CONNECTION_TIMEOUT: 60 + +jobs: + test-warm-containers: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install SAM CLI in dev mode + run: pip install -e ".[dev]" + + - name: Run warm containers tests + run: | + pytest -vv --reruns 3 \ + tests/integration/local/start_api/test_start_api.py::TestWarmContainersHandlesSigTerm \ + tests/integration/local/start_api/test_start_api.py::TestWarmContainersInitialization \ + tests/integration/local/start_api/test_start_api.py::TestWarmContainersMultipleInvoke \ + tests/integration/local/start_lambda/test_start_lambda.py::TestWarmContainersHandlesSigTermInterrupt \ + --json-report --json-report-file=test-report.json + + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-report + path: test-report.json diff --git a/tests/integration/local/start_api/test_start_api.py b/tests/integration/local/start_api/test_start_api.py index e53b2540126..5ddd7bc8a70 100644 --- a/tests/integration/local/start_api/test_start_api.py +++ b/tests/integration/local/start_api/test_start_api.py @@ -2220,35 +2220,26 @@ def setUp(self): def count_running_containers(self): """Count containers created by this test using Docker client directly.""" - # Use Docker client to find containers with SAM CLI labels try: - # Get running containers with SAM CLI lambda container label sam_containers = self.docker_client.containers.list( all=False, filters={"label": "sam.cli.container.type=lambda"} ) - # Filter by our test's mode environment variable if possible test_containers = [] for container in sam_containers: try: container.reload() env_vars = container.attrs.get("Config", {}).get("Env", []) for env_var in env_vars: - if env_var.startswith("MODE=") and self.mode_env_variable in env_var: + if env_var == f"MODE={self.mode_env_variable}": test_containers.append(container) break except Exception: continue - # If we found containers with our mode variable, return that count - if test_containers: - return len(test_containers) + return len(test_containers) - # Otherwise, return all SAM containers (fallback) - return len(sam_containers) - - except Exception as e: - # If we can't access Docker client, fall back to 0 + except Exception: return 0 def _parse_container_ids_from_output(self): diff --git a/tests/integration/local/start_lambda/test_start_lambda.py b/tests/integration/local/start_lambda/test_start_lambda.py index c2765887b52..3039ee2a82c 100644 --- a/tests/integration/local/start_lambda/test_start_lambda.py +++ b/tests/integration/local/start_lambda/test_start_lambda.py @@ -403,35 +403,26 @@ def setUp(self): def count_running_containers(self): """Count containers created by this test using Docker client directly.""" - # Use Docker client to find containers with SAM CLI labels try: - # Get running containers with SAM CLI lambda container label sam_containers = self.docker_client.containers.list( all=False, filters={"label": "sam.cli.container.type=lambda"} ) - # Filter by our test's mode environment variable if possible test_containers = [] for container in sam_containers: try: container.reload() env_vars = container.attrs.get("Config", {}).get("Env", []) for env_var in env_vars: - if env_var.startswith("MODE=") and self.mode_env_variable in env_var: + if env_var == f"MODE={self.mode_env_variable}": test_containers.append(container) break except Exception: continue - # If we found containers with our mode variable, return that count - if test_containers: - return len(test_containers) + return len(test_containers) - # Otherwise, return all SAM containers (fallback) - return len(sam_containers) - - except Exception as e: - # If we can't access Docker client, fall back to 0 + except Exception: return 0 def _parse_container_ids_from_output(self): From aebf7185ba3caeb71564434f3a86975d79975393 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Fri, 13 Feb 2026 03:28:41 +0000 Subject: [PATCH 4/6] fix: add AWS_DEFAULT_REGION and use -k pattern for parameterized tests --- .github/workflows/test-warm-containers-fix.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-warm-containers-fix.yml b/.github/workflows/test-warm-containers-fix.yml index 43d96680916..94ba200fa0e 100644 --- a/.github/workflows/test-warm-containers-fix.yml +++ b/.github/workflows/test-warm-containers-fix.yml @@ -9,6 +9,7 @@ permissions: contents: read env: + AWS_DEFAULT_REGION: us-east-1 SAM_CLI_DEV: 1 SAM_CLI_TELEMETRY: 0 SAM_CLI_CONTAINER_CONNECTION_TIMEOUT: 60 @@ -29,10 +30,9 @@ jobs: - name: Run warm containers tests run: | pytest -vv --reruns 3 \ - tests/integration/local/start_api/test_start_api.py::TestWarmContainersHandlesSigTerm \ - tests/integration/local/start_api/test_start_api.py::TestWarmContainersInitialization \ - tests/integration/local/start_api/test_start_api.py::TestWarmContainersMultipleInvoke \ - tests/integration/local/start_lambda/test_start_lambda.py::TestWarmContainersHandlesSigTermInterrupt \ + -k "TestWarmContainers" \ + tests/integration/local/start_api/test_start_api.py \ + tests/integration/local/start_lambda/test_start_lambda.py \ --json-report --json-report-file=test-report.json - name: Upload test report From 5f8552cc1bdcdfc6ae89e17241bc599eb5ed214d Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Fri, 13 Feb 2026 03:30:05 +0000 Subject: [PATCH 5/6] fix: add BY_CANARY=true to enable Docker tests on CI --- .github/workflows/test-warm-containers-fix.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-warm-containers-fix.yml b/.github/workflows/test-warm-containers-fix.yml index 94ba200fa0e..6e23b247ee5 100644 --- a/.github/workflows/test-warm-containers-fix.yml +++ b/.github/workflows/test-warm-containers-fix.yml @@ -13,6 +13,7 @@ env: SAM_CLI_DEV: 1 SAM_CLI_TELEMETRY: 0 SAM_CLI_CONTAINER_CONNECTION_TIMEOUT: 60 + BY_CANARY: true jobs: test-warm-containers: From 80fbb7b1e704ebf52bd2c718be95e79b6328dbb8 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Fri, 13 Feb 2026 03:33:20 +0000 Subject: [PATCH 6/6] fix: exclude RemoteLayers tests that need AWS credentials --- .github/workflows/test-warm-containers-fix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-warm-containers-fix.yml b/.github/workflows/test-warm-containers-fix.yml index 6e23b247ee5..aa14c6c162e 100644 --- a/.github/workflows/test-warm-containers-fix.yml +++ b/.github/workflows/test-warm-containers-fix.yml @@ -31,7 +31,7 @@ jobs: - name: Run warm containers tests run: | pytest -vv --reruns 3 \ - -k "TestWarmContainers" \ + -k "TestWarmContainers and not RemoteLayers" \ tests/integration/local/start_api/test_start_api.py \ tests/integration/local/start_lambda/test_start_lambda.py \ --json-report --json-report-file=test-report.json