fix: build emulator image and copy binary to it (#8484)

Author: bchampp

samcli/local/docker/durable_functions_emulator_container.py

@@ -7,6 +7,7 @@
 import time
 from http import HTTPStatus
 from pathlib import Path
+from tempfile import NamedTemporaryFile
 from typing import Optional
 
 import docker
@@ -15,7 +16,8 @@
 
 from samcli.lib.build.utils import _get_host_architecture
 from samcli.lib.clients.lambda_client import DurableFunctionsClient
-from samcli.local.docker.utils import get_validated_container_client, is_image_current
+from samcli.lib.utils.tar import create_tarball
+from samcli.local.docker.utils import get_tar_filter_for_windows, get_validated_container_client, is_image_current
 
 LOG = logging.getLogger(__name__)
 
@@ -27,6 +29,7 @@ class DurableFunctionsEmulatorContainer:
 
     _RAPID_SOURCE_PATH = Path(__file__).parent.joinpath("..", "rapid").resolve()
     _EMULATOR_IMAGE = "public.ecr.aws/ubuntu/ubuntu:24.04"
+    _EMULATOR_IMAGE_PREFIX = "samcli/durable-execution-emulator"
     _CONTAINER_NAME = "sam-durable-execution-emulator"
     _EMULATOR_DATA_DIR_NAME = ".durable-executions-local"
     _EMULATOR_DEFAULT_STORE_TYPE = "sqlite"
@@ -190,6 +193,62 @@ def _get_emulator_binary_name(self):
         arch = _get_host_architecture()
         return f"aws-durable-execution-emulator-{arch}"
 
+    def _generate_emulator_dockerfile(self, emulator_binary_name: str) -> str:
+        """Generate Dockerfile content for emulator image."""
+        return (
+            f"FROM {self._EMULATOR_IMAGE}\n"
+            f"COPY {emulator_binary_name} /usr/local/bin/{emulator_binary_name}\n"
+            f"RUN chmod +x /usr/local/bin/{emulator_binary_name}\n"
+        )
+
+    def _get_emulator_image_tag(self, emulator_binary_name: str) -> str:
+        """Get the Docker image tag for the emulator."""
+        return f"{self._EMULATOR_IMAGE_PREFIX}:{emulator_binary_name}"
+
+    def _build_emulator_image(self):
+        """Build Docker image with emulator binary."""
+        emulator_binary_name = self._get_emulator_binary_name()
+        binary_path = self._RAPID_SOURCE_PATH / emulator_binary_name
+
+        if not binary_path.exists():
+            raise RuntimeError(f"Durable Functions Emulator binary not found at {binary_path}")
+
+        image_tag = self._get_emulator_image_tag(emulator_binary_name)
+
+        # Check if image already exists
+        try:
+            self._docker_client.images.get(image_tag)
+            LOG.debug(f"Emulator image {image_tag} already exists")
+            return image_tag
+        except docker.errors.ImageNotFound:
+            LOG.debug(f"Building emulator image {image_tag}")
+
+        # Generate Dockerfile content
+        dockerfile_content = self._generate_emulator_dockerfile(emulator_binary_name)
+
+        # Write Dockerfile to temp location and build image
+        with NamedTemporaryFile(mode="w", suffix="_Dockerfile") as dockerfile:
+            dockerfile.write(dockerfile_content)
+            dockerfile.flush()
+
+            # Prepare tar paths for build context
+            tar_paths = {
+                dockerfile.name: "Dockerfile",
+                str(binary_path): emulator_binary_name,
+            }
+
+            # Use shared tar filter for Windows compatibility
+            tar_filter = get_tar_filter_for_windows()
+
+            # Build image using create_tarball utility
+            with create_tarball(tar_paths, tar_filter=tar_filter, dereference=True) as tarballfile:
+                try:
+                    self._docker_client.images.build(fileobj=tarballfile, custom_context=True, tag=image_tag, rm=True)
+                    LOG.info(f"Built emulator image {image_tag}")
+                    return image_tag
+                except Exception as e:
+                    raise ClickException(f"Failed to build emulator image: {e}")
+
     def _pull_image_if_needed(self):
         """Pull the emulator image if it doesn't exist locally or is out of date."""
         try:
@@ -218,9 +277,6 @@ def start(self):
             return
 
         emulator_binary_name = self._get_emulator_binary_name()
-        binary_path = self._RAPID_SOURCE_PATH / emulator_binary_name
-        if not binary_path.exists():
-            raise RuntimeError(f"Durable Functions Emulator binary not found at {binary_path}")
 
         """
         Create persistent volume for execution data to be stored in.
@@ -231,16 +287,15 @@ def start(self):
         os.makedirs(emulator_data_dir, exist_ok=True)
 
         volumes = {
-            str(self._RAPID_SOURCE_PATH): {"bind": "/usr/local/bin", "mode": "ro"},
             emulator_data_dir: {"bind": "/tmp/.durable-executions-local", "mode": "rw"},
         }
 
-        # Pull the image if needed
-        self._pull_image_if_needed()
+        # Build image with emulator binary
+        image_tag = self._build_emulator_image()
 
         LOG.debug(f"Creating container with name={self._container_name}, port={self.port}")
         self.container = self._docker_client.containers.create(
-            image=self._EMULATOR_IMAGE,
+            image=image_tag,
             command=[f"/usr/local/bin/{emulator_binary_name}", "--host", "0.0.0.0", "--port", str(self.port)],
             name=self._container_name,
             ports={f"{self.port}/tcp": self.port},

samcli/local/docker/lambda_image.py

@@ -26,7 +26,12 @@
 from samcli.lib.utils.stream_writer import StreamWriter
 from samcli.lib.utils.tar import create_tarball
 from samcli.local.common.file_lock import FileLock, cleanup_stale_locks
-from samcli.local.docker.utils import get_docker_platform, get_rapid_name, get_validated_container_client
+from samcli.local.docker.utils import (
+    get_docker_platform,
+    get_rapid_name,
+    get_tar_filter_for_windows,
+    get_validated_container_client,
+)
 
 LOG = logging.getLogger(__name__)
 
@@ -409,15 +414,8 @@ def _build_image(self, base_image, docker_tag, layers, architecture, stream=None
             for layer in layers:
                 tar_paths[layer.codeuri] = "/" + layer.name
 
-            # Set permission for all the files in the tarball to 500(Read and Execute Only)
-            # This is need for systems without unix like permission bits(Windows) while creating a unix image
-            # Without setting this explicitly, tar will default the permission to 666 which gives no execute permission
-            def set_item_permission(tar_info):
-                tar_info.mode = 0o500
-                return tar_info
-
-            # Set only on Windows, unix systems will preserve the host permission into the tarball
-            tar_filter = set_item_permission if platform.system().lower() == "windows" else None
+            # Use shared tar filter for Windows compatibility
+            tar_filter = get_tar_filter_for_windows()
 
             with create_tarball(tar_paths, tar_filter=tar_filter, dereference=True) as tarballfile:
                 try:

samcli/local/docker/utils.py

@@ -5,6 +5,7 @@
 import logging
 import os
 import pathlib
+import platform
 import posixpath
 import random
 import re
@@ -138,6 +139,27 @@ def get_validated_container_client():
     return ContainerClientFactory.create_client()
 
 
+def get_tar_filter_for_windows():
+    """
+    Get tar filter function for Windows compatibility.
+
+    Sets permission for all files in the tarball to 500 (Read and Execute Only).
+    This is needed for systems without unix-like permission bits (Windows) while creating a unix image.
+    Without setting this explicitly, tar will default the permission to 666 which gives no execute permission.
+
+    Returns
+    -------
+    callable or None
+        Filter function for Windows, None for Unix systems
+    """
+
+    def set_item_permission(tar_info):
+        tar_info.mode = 0o500
+        return tar_info
+
+    return set_item_permission if platform.system().lower() == "windows" else None
+
+
 def is_image_current(docker_client: docker.DockerClient, image_name: str) -> bool:
     """
     Check if local image is up-to-date with remote by comparing digests.

tests/unit/local/docker/test_durable_functions_emulator_container.py

@@ -248,28 +248,46 @@ def test_binary_selection_by_architecture(self, arch, expected_binary, mock_get_
             ),
         ]
     )
-    @patch("samcli.local.docker.durable_functions_emulator_container.is_image_current")
+    @patch("samcli.local.docker.durable_functions_emulator_container._get_host_architecture")
     @patch("os.makedirs")
     @patch("os.getcwd")
+    @patch("pathlib.Path.exists")
     def test_create_container(
-        self, name, env_vars, expected_port, expected_store, expected_scale, mock_getcwd, mock_makedirs, mock_is_current
+        self,
+        name,
+        env_vars,
+        expected_port,
+        expected_store,
+        expected_scale,
+        mock_path_exists,
+        mock_getcwd,
+        mock_makedirs,
+        mock_get_host_arch,
     ):
         """Test container creation with all configuration permutations"""
-        mock_is_current.return_value = True
+        mock_get_host_arch.return_value = "x86_64"
         test_dir = "/test/dir"
         mock_getcwd.return_value = test_dir
+        mock_path_exists.return_value = True
+
+        # Mock image already exists
+        mock_image = Mock()
+        self.mock_docker_client.images.get.return_value = mock_image
 
         with patch.dict("os.environ", env_vars, clear=True):
             container = self._create_container()
+            container._RAPID_SOURCE_PATH = Path(__file__).parent
             container._wait_for_ready = Mock()
             container.start()
 
             # Verify container was created
             self.mock_docker_client.containers.create.assert_called_once()
             call_args = self.mock_docker_client.containers.create.call_args
 
-            # Verify image and working directory
-            self.assertEqual(call_args.kwargs["image"], DurableFunctionsEmulatorContainer._EMULATOR_IMAGE)
+            # Verify built image is used
+            self.assertEqual(
+                call_args.kwargs["image"], "samcli/durable-execution-emulator:aws-durable-execution-emulator-x86_64"
+            )
             self.assertEqual(call_args.kwargs["working_dir"], "/tmp/.durable-executions-local")
 
             # Verify port configuration
@@ -306,6 +324,132 @@ def test_start_raises_error_when_binary_not_found(self):
             container.start()
         self.assertIn("Durable Functions Emulator binary not found", str(context.exception))
 
+    @parameterized.expand(
+        [
+            (
+                "x86_64",
+                "aws-durable-execution-emulator-x86_64",
+                "samcli/durable-execution-emulator:aws-durable-execution-emulator-x86_64",
+            ),
+            (
+                "arm64",
+                "aws-durable-execution-emulator-arm64",
+                "samcli/durable-execution-emulator:aws-durable-execution-emulator-arm64",
+            ),
+        ]
+    )
+    @patch("samcli.local.docker.durable_functions_emulator_container._get_host_architecture")
+    @patch("samcli.local.docker.durable_functions_emulator_container.create_tarball")
+    @patch("samcli.local.docker.durable_functions_emulator_container.get_tar_filter_for_windows")
+    @patch("builtins.open", new_callable=mock_open)
+    @patch("os.unlink")
+    @patch("pathlib.Path.exists")
+    def test_build_emulator_image_creates_new_image(
+        self,
+        arch,
+        binary_name,
+        expected_tag,
+        mock_path_exists,
+        mock_unlink,
+        mock_file,
+        mock_tar_filter,
+        mock_create_tarball,
+        mock_get_host_arch,
+    ):
+        """Test building emulator image when it doesn't exist, including dockerfile generation and image tag"""
+        mock_get_host_arch.return_value = arch
+        mock_tar_filter.return_value = None
+        mock_tarball = Mock()
+        mock_create_tarball.return_value.__enter__.return_value = mock_tarball
+        mock_path_exists.return_value = True
+
+        # Mock image doesn't exist
+        self.mock_docker_client.images.get.side_effect = docker.errors.ImageNotFound("not found")
+        mock_build_result = Mock()
+        self.mock_docker_client.images.build.return_value = mock_build_result
+
+        container = self._create_container()
+        container._RAPID_SOURCE_PATH = Path(__file__).parent
+
+        result = container._build_emulator_image()
+
+        # Verify image tag generation
+        self.assertEqual(result, expected_tag)
+        tag = container._get_emulator_image_tag(binary_name)
+        self.assertEqual(tag, expected_tag)
+
+        # Verify dockerfile generation
+        dockerfile = container._generate_emulator_dockerfile(binary_name)
+        self.assertIn(f"FROM {container._EMULATOR_IMAGE}", dockerfile)
+        self.assertIn(f"COPY {binary_name} /usr/local/bin/{binary_name}", dockerfile)
+        self.assertIn(f"RUN chmod +x /usr/local/bin/{binary_name}", dockerfile)
+
+        # Verify image was built
+        self.mock_docker_client.images.build.assert_called_once()
+        build_call = self.mock_docker_client.images.build.call_args
+        self.assertEqual(build_call.kwargs["tag"], expected_tag)
+        self.assertTrue(build_call.kwargs["rm"])
+        self.assertTrue(build_call.kwargs["custom_context"])
+
+        # Verify tarball was created with correct filter
+        mock_create_tarball.assert_called_once()
+
+    @parameterized.expand(
+        [
+            ("x86_64", "samcli/durable-execution-emulator:aws-durable-execution-emulator-x86_64"),
+            ("arm64", "samcli/durable-execution-emulator:aws-durable-execution-emulator-arm64"),
+        ]
+    )
+    @patch("samcli.local.docker.durable_functions_emulator_container._get_host_architecture")
+    @patch("pathlib.Path.exists")
+    def test_build_emulator_image_reuses_existing(self, arch, expected_tag, mock_path_exists, mock_get_host_arch):
+        """Test that existing image is reused without rebuilding"""
+        mock_get_host_arch.return_value = arch
+        mock_path_exists.return_value = True
+        mock_image = Mock()
+        self.mock_docker_client.images.get.return_value = mock_image
+
+        container = self._create_container()
+        container._RAPID_SOURCE_PATH = Path(__file__).parent
+
+        result = container._build_emulator_image()
+
+        # Verify image was not built
+        self.mock_docker_client.images.build.assert_not_called()
+        self.assertEqual(result, expected_tag)
+
+    @parameterized.expand(
+        [
+            ("x86_64", "samcli/durable-execution-emulator:aws-durable-execution-emulator-x86_64"),
+            ("arm64", "samcli/durable-execution-emulator:aws-durable-execution-emulator-arm64"),
+        ]
+    )
+    @patch("samcli.local.docker.durable_functions_emulator_container._get_host_architecture")
+    @patch("os.makedirs")
+    @patch("os.getcwd")
+    @patch("pathlib.Path.exists")
+    def test_start_uses_built_image(
+        self, arch, expected_tag, mock_path_exists, mock_getcwd, mock_makedirs, mock_get_host_arch
+    ):
+        """Test that start() uses the built image instead of base image"""
+        mock_get_host_arch.return_value = arch
+        mock_getcwd.return_value = "/test/dir"
+        mock_path_exists.return_value = True
+
+        # Mock image already exists
+        mock_image = Mock()
+        self.mock_docker_client.images.get.return_value = mock_image
+
+        container = self._create_container()
+        container._RAPID_SOURCE_PATH = Path(__file__).parent
+        container._wait_for_ready = Mock()
+
+        container.start()
+
+        # Verify container was created with built image tag
+        call_args = self.mock_docker_client.containers.create.call_args
+        self.assertEqual(call_args.kwargs["image"], expected_tag)
+
     @parameterized.expand(
         [
             # (name, image_exists, is_current, should_pull)