Feature - ECS Service Connect (#745)

Author: JohnPreston

docs/syntax/compose_x/ecs.details/network.rst

@@ -14,6 +14,7 @@ services.x-network
     services:
       serviceA:
         x-network:
+          x-ecs_connect: {}
           AssignPublicIp: bool
           Ingress: {}
           x-cloudmap: {}
@@ -41,6 +42,103 @@ This flag allows to assign an Elastic IP to the container when using ``awsvpc``
     To select which subnets to place the services, see :ref:`compose_networks_syntax_reference`
 
 
+x-ecs_connect (1.1.0)
+======================
+
+This configuration section allows you to define ECS Service Connect configuration.
+It's made up of two options, `Properties` and `MacroParameters`
+
+`Properties` must match exactly the `ECS Service Connect properties`_ and must be all valid to work.
+
+.. attention::
+
+    No changes to input or validation will be made when set. Be sure to have everything valid.
+
+`MacroParameters` however, is an attempt at creating a shorthand syntax to this.
+
+service connect - client only
+------------------------------
+
+You might have applications that you want to act only as clients to other services. This will only tell ECS to make sure
+to provision the Service Connect sidecar which will be there to handle the proxy-ing to server services.
+
+To enable the client config, you simply need to enable the feature as show below
+
+.. code-block::
+
+    x-cloudmap:
+      PrivateNamespace:
+        Name: compose-x.internal
+
+    services:
+      yelb-ui:
+        x-network:
+          AssignPublicIp: true
+          x-ecs_connect:
+            MacroParameters:
+              x-cloudmap: PrivateNamespace
+          Ingress:
+            ExtSources:
+              - IPv4: 0.0.0.0/0
+                Name: ANY
+
+service connect - server
+----------------------------
+
+For services that you want to act as client & server, you need to declare which ports you want to declare to Service Connect.
+That's mandatory.
+
+For example, we have the following two services: appserver will act as both a client and a server. It will serve requests
+for our yelb-ui service (the client above), and a client to the redis-server
+
+.. code-block::
+
+    x-cloudmap:
+      PrivateNamespace:
+        Name: compose-x.internal
+
+    services:
+      yelb-appserver:
+        image: mreferre/yelb-appserver:0.7
+        depends_on:
+          - redis-server
+        ports:
+          - 4567:4567
+        environment:
+          redishost: redis-server
+        x-network:
+          Ingress:
+            Services:
+              - Name: yelb-ui
+          x-ecs_connect:
+            MacroParameters:
+              service_ports:
+                tcp_4567:
+                  DnsName: yelb-appserver
+                  CloudMapServiceName: yelb-appserver
+              x-cloudmap: PrivateNamespace
+
+
+      redis-server:
+        image: redis:4.0.2
+        ports:
+          - 6379:6379
+        x-network:
+          x-ecs_connect:
+            MacroParameters:
+              service_ports:
+                tcp_6379:
+                  DnsName: redis-server
+                  CloudMapServiceName: redis-server
+              x-cloudmap: PrivateNamespace
+          Ingress:
+            Services:
+              - Name: yelb-appserver
+
+.. hint::
+
+    See `the full connect example`_ uses to perform functional testing of the feature.
+
 Ingress
 ======================
 
@@ -148,3 +246,6 @@ Definition
 -----------
 
 .. literalinclude:: ../../../../ecs_composex/specs/services.x-network.spec.json
+
+.. _ECS Service Connect properties: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ecs-service-serviceconnectconfiguration.html
+.. _the full connect example: https://github.com/compose-x/ecs_composex/tree/main/use-cases/yelb.yaml

ecs_composex/cloudmap/cloudmap_params.py

@@ -48,6 +48,15 @@
     AllowedPattern=ZONES_PATTERN.pattern,
 )
 
+PRIVATE_NAMESPACE_ARN_T: str = "PrivateNamespaceArn"
+PRIVATE_NAMESPACE_ARN: Parameter = Parameter(
+    PRIVATE_NAMESPACE_ARN_T,
+    group_label=LABEL,
+    return_value="Arn",
+    Type="String",
+)
+
+
 ECS_SERVICE_NAMESPACE_SERVICE_ID_T = "EcsCloudMapServiceName"
 ECS_SERVICE_NAMESPACE_SERVICE_ID = Parameter(
     ECS_SERVICE_NAMESPACE_SERVICE_ID_T,

ecs_composex/cloudmap/cloudmap_stack.py

@@ -43,6 +43,7 @@
     MOD_KEY,
     PRIVATE_DNS_ZONE_ID,
     PRIVATE_DNS_ZONE_NAME,
+    PRIVATE_NAMESPACE_ARN,
     PRIVATE_NAMESPACE_ID,
 )
 from .cloudmap_x_resources import handle_resource_cloudmap_settings
@@ -99,6 +100,12 @@ def init_outputs(self):
                 self.zone_name,
                 False,
             ),
+            PRIVATE_NAMESPACE_ARN: (
+                f"{self.logical_name}{PRIVATE_NAMESPACE_ARN.return_value}",
+                self.cfn_resource,
+                GetAtt,
+                PRIVATE_NAMESPACE_ARN.return_value,
+            ),
         }
 
     @property

ecs_composex/common/settings.py

@@ -281,6 +281,7 @@ def get_resource_attribute(self, compose_resource_arn: str) -> tuple:
             ]
             return resource, parameter
         except LookupError:
+            print(f"Not found {compose_resource_arn}")
             return None, None
 
     @property

ecs_composex/compose/compose_services/__init__.py

@@ -105,6 +105,7 @@ def __init__(
         self.x_scaling = set_else_none("x-scaling", self.definition, None, False)
         self.x_network = set_else_none("x-network", self.definition, None, False)
         self.x_cloudmap = set_else_none("x-cloudmap", self.x_network, None, False)
+        self.x_ecs_connect = set_else_none("x-ecs_connect", self.x_network, None)
         self.x_ecs = set_else_none("x-ecs", self.definition, {})
         self.ecr_config = set_else_none("x-ecr", self.definition, None)
         self.x_ecr = set_else_none("x-ecr", self.definition, {})

ecs_composex/ecs/ecs_family/__init__.py

@@ -12,6 +12,7 @@
 if TYPE_CHECKING:
     from troposphere.ecs import Service as CfnService
     from ecs_composex.common.settings import ComposeXSettings
+    from ecs_composex.ecs.ecs_service import EcsService
 
 import re
 from itertools import chain
@@ -72,8 +73,8 @@ class ComposeFamily:
     """
 
     def __init__(self, services: list[ComposeService], family_name):
-        self._compose_services = services
-        self.ordered_services = services
+        self._compose_services: list[ComposeService] = services
+        self.ordered_services: list[ComposeService] = services
         self.managed_sidecars = []
         self.name = family_name
         self.family_hostname = self.name.replace("_", "-").lower()
@@ -92,7 +93,7 @@ def __init__(self, services: list[ComposeService], family_name):
         self.task_definition = None
         self.service_tags = None
         self.enable_execute_command = False
-        self.ecs_service = None
+        self.ecs_service: EcsService | None = None
         self.runtime_cpu_arch = None
         self.runtime_os_family = None
         self.outputs = []
@@ -103,9 +104,9 @@ def __init__(self, services: list[ComposeService], family_name):
         self.iam_manager = TaskIam(self)
         self.iam_manager.init_update_policies()
         self.service_scaling = None
-        self.service_networking = None
+        self.service_networking: ServiceNetworking | None = None
         self.task_compute = None
-        self.service_compute = ServiceCompute(self)
+        self.service_compute: ServiceCompute = ServiceCompute(self)
         self.set_enable_execute_command()
         set_family_hostname(self)
 
@@ -447,41 +448,29 @@ def init_network_settings(
             self.service_networking.ingress.associate_ext_ingress_rules(self.template)
             self.service_networking.add_self_ingress()
 
-    def finalize_family_settings(self):
+    def finalize_family_settings(self, settings: ComposeXSettings):
         """
         Once all services have been added, we add the sidecars and deal with appropriate permissions and settings
         Will add xray / prometheus sidecars
         """
-        from .family_helpers import set_service_dependency_on_all_iam_policies
+        from ecs_composex.ecs.ecs_family.family_helpers import (
+            set_service_dependency_on_all_iam_policies,
+        )
+        from ecs_composex.ecs.ecs_family.family_helpers.compute_finalizers import (
+            finalize_family_compute,
+            finalize_scaling_settings,
+        )
+        from ecs_composex.ecs.ecs_family.family_helpers.network_finalizers import (
+            finalize_lb_settings,
+            finalize_network_settings,
+        )
 
-        self.add_containers_images_cfn_parameters()
-        self.task_compute.set_task_compute_parameter()
-        self.task_compute.unlock_compute_for_main_container()
-        if self.service_compute.ecs_capacity_providers:
-            self.service_compute.apply_capacity_providers_to_service(
-                self.service_compute.ecs_capacity_providers
-            )
+        finalize_network_settings(self, settings)
+        finalize_family_compute(self)
 
         set_service_dependency_on_all_iam_policies(self)
-        if self.service_compute.launch_type == "EXTERNAL":
-            if hasattr(self.service_definition, "LoadBalancers"):
-                setattr(self.service_definition, "LoadBalancers", NoValue)
-            if hasattr(self.service_definition, "ServiceRegistries"):
-                setattr(self.service_definition, "ServiceRegistries", NoValue)
-            for container in self.task_definition.ContainerDefinitions:
-                if hasattr(container, "LinuxParameters"):
-                    parameters = getattr(container, "LinuxParameters")
-                    setattr(parameters, "InitProcessEnabled", False)
-        if (
-            self.service_definition
-            and self.service_definition.title in self.template.resources
-        ) and (
-            self.service_scaling
-            and self.service_scaling.scalable_target
-            and self.service_scaling.scalable_target.title
-            not in self.template.resources
-        ):
-            self.template.add_resource(self.service_scaling.scalable_target)
+        finalize_lb_settings(self)
+        finalize_scaling_settings(self)
         self.generate_outputs()
         service_configs = [
             [0, service]

…omposex/ecs/ecs_family/family_helpers.py …cs/ecs_family/family_helpers/__init__.pyecs_composex/ecs/ecs_family/family_helpers.py renamed to ecs_composex/ecs/ecs_family/family_helpers/__init__.py ecs_composex/ecs/ecs_family/family_helpers.py renamed to ecs_composex/ecs/ecs_family/family_helpers/__init__.py

@@ -0,0 +1,36 @@
+#  SPDX-License-Identifier: MPL-2.0
+#  Copyright 2024 John Mille <john@compose-x.io>
+
+"""Functions to finalize the family compute & scaling settings"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ecs_composex.ecs.ecs_family import ComposeFamily
+
+
+def finalize_family_compute(family: ComposeFamily) -> None:
+    """Finalizes the family compute settings"""
+    family.add_containers_images_cfn_parameters()
+    family.task_compute.set_task_compute_parameter()
+    family.task_compute.unlock_compute_for_main_container()
+    if family.service_compute.ecs_capacity_providers:
+        family.service_compute.apply_capacity_providers_to_service(
+            family.service_compute.ecs_capacity_providers
+        )
+
+
+def finalize_scaling_settings(family: ComposeFamily) -> None:
+    """If family has scaling target configured, ensures that the scalable target gets created."""
+    if (
+        family.service_definition
+        and family.service_definition.title in family.template.resources
+    ) and (
+        family.service_scaling
+        and family.service_scaling.scalable_target
+        and family.service_scaling.scalable_target.title
+        not in family.template.resources
+    ):
+        family.template.add_resource(family.service_scaling.scalable_target)

ecs_composex/ecs/ecs_family/family_helpers/compute_finalizers.py

@@ -0,0 +1,47 @@
+#  SPDX-License-Identifier: MPL-2.0
+#  Copyright 2024 John Mille <john@compose-x.io>
+
+"""Functions to finalize the family networking settings"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ecs_composex.common.settings import ComposeXSettings
+    from ecs_composex.ecs.ecs_family import ComposeFamily
+
+from troposphere import NoValue
+
+
+def finalize_network_settings(
+    family: ComposeFamily, settings: ComposeXSettings
+) -> None:
+    """
+    Evaluates the ECS Connect settings to be configured by the service.
+    If there is a configuration to be set, ensures it's set on the ECS Service definition.
+    """
+    family.service_networking.set_ecs_connect(settings)
+    if family.service_networking.ecs_connect_config and family.ecs_service:
+        setattr(
+            family.ecs_service.ecs_service,
+            "ServiceConnectConfiguration",
+            family.service_networking.ecs_connect_config,
+        )
+
+
+def finalize_lb_settings(family: ComposeFamily) -> None:
+    """
+    Ensures that the LoadBalancers & ServiceRegistries (LB & CloudMap) are set appropriately based on
+    the deployment settings. Especially, resets properties if the service is deployed to ECS Anywhere.
+    Ensures correctness of LinuxParameters for each of the services.
+    """
+    if family.service_compute.launch_type == "EXTERNAL":
+        if hasattr(family.service_definition, "LoadBalancers"):
+            setattr(family.service_definition, "LoadBalancers", NoValue)
+        if hasattr(family.service_definition, "ServiceRegistries"):
+            setattr(family.service_definition, "ServiceRegistries", NoValue)
+        for container in family.task_definition.ContainerDefinitions:
+            if hasattr(container, "LinuxParameters"):
+                parameters = getattr(container, "LinuxParameters")
+                setattr(parameters, "InitProcessEnabled", False)