k8s service: connect with overlay IP address instead of pod IP

Author: boddumanohar

src/deployment/__init__.py

@@ -46,7 +46,7 @@
 from ._util import deployment_namespace
 from .deployment import DeploymentParameters
 from .grafana import create_vela_grafana_obj, delete_vela_grafana_obj
-from .kubernetes import KubernetesService
+from .kubernetes import KubernetesService, get_neon_vm
 from .kubernetes._util import custom_api_client
 from .settings import get_settings
 from .simplyblock_api import create_simplyblock_api
@@ -75,6 +75,8 @@
 AUTOSCALER_PVC_SUFFIX = "-block-data"
 _LOAD_BALANCER_TIMEOUT_SECONDS = float(600)
 _LOAD_BALANCER_POLL_INTERVAL_SECONDS = float(2)
+_OVERLAY_IP_TIMEOUT_SECONDS = float(300)
+_OVERLAY_IP_POLL_INTERVAL_SECONDS = float(5)
 DNSRecordType = Literal["AAAA", "CNAME"]
 DATABASE_DNS_RECORD_TYPE: Literal["AAAA"] = "AAAA"
 
@@ -157,6 +159,68 @@ def branch_service_name(component: str) -> str:
     return f"{_release_fullname()}-{component}"
 
 
+async def _wait_for_autoscaler_overlay_ip(namespace: str, vm_name: str) -> str:
+    loop = asyncio.get_running_loop()
+    deadline = loop.time() + _OVERLAY_IP_TIMEOUT_SECONDS
+    last_error: Exception | None = None
+    logger.info("Waiting for overlay IP for autoscaler VM %s/%s", namespace, vm_name)
+
+    while True:
+        try:
+            vm = await get_neon_vm(namespace, vm_name)
+        except (VelaKubernetesError, RuntimeError) as exc:
+            last_error = exc
+            vm = None
+
+        if vm:
+            overlay_ip = (vm.status.extra_net_ip or "").strip()
+            if overlay_ip:
+                logger.info(
+                    "Autoscaler VM %s/%s overlay network %s is ready",
+                    namespace,
+                    vm_name,
+                    overlay_ip,
+                )
+                return overlay_ip
+
+        if loop.time() >= deadline:
+            message = f"Timed out waiting for overlay IP for autoscaler VM {vm_name} in namespace {namespace}"
+            if last_error is not None:
+                raise VelaDeploymentError(message) from last_error
+            raise VelaDeploymentError(message)
+
+        await asyncio.sleep(_OVERLAY_IP_POLL_INTERVAL_SECONDS)
+
+
+def _overlay_service_specs() -> list[tuple[str, int, str]]:
+    return [
+        (branch_service_name("db"), 5432, "postgres"),
+        (branch_service_name("pgbouncer"), 6432, "pgbouncer"),
+        (branch_service_name("rest"), 3000, "http"),
+        (branch_service_name("storage"), 5000, "http"),
+        (branch_service_name("meta"), 8080, "http"),
+        (branch_service_name("pgexporter"), 9187, "http"),
+    ]
+
+
+async def _ensure_autoscaler_overlay_endpoint_slices(namespace: str, overlay_ip: str) -> None:
+    for service_name, port, port_name in _overlay_service_specs():
+        await kube_service.ensure_endpoint_slice(
+            namespace=namespace,
+            slice_name=service_name,
+            service_name=service_name,
+            address=overlay_ip,
+            port=port,
+            port_name=port_name,
+        )
+
+
+async def _initialize_autoscaler_overlay_endpoints(namespace: str) -> None:
+    vm_name = _autoscaler_vm_name()
+    overlay_ip = await _wait_for_autoscaler_overlay_ip(namespace, vm_name)
+    await _ensure_autoscaler_overlay_endpoint_slices(namespace, overlay_ip)
+
+
 def _build_storage_class_manifest(*, storage_class_name: str, iops: int, base_storage_class: Any) -> dict[str, Any]:
     provisioner = getattr(base_storage_class, "provisioner", None)
     if not provisioner:
@@ -486,6 +550,7 @@ async def create_vela_config(
                 stderr=subprocess.PIPE,
                 text=True,
             )
+            await _initialize_autoscaler_overlay_endpoints(namespace)
         except subprocess.CalledProcessError as e:
             logger.exception(f"Failed to create deployment: {e.stderr}")
             release_name = _release_name()

src/deployment/charts/vela/templates/_helpers.tpl

@@ -44,13 +44,6 @@ Return the PgBouncer config map name that lives next to the autoscaler VM.
 {{- printf "%s-pgbouncer" (include "vela.autoscaler.name" .) | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
-{{/*
-Selector label used to identify the autoscaler VM pods.
-*/}}
-{{- define "vela.autoscaler.selectorLabel" -}}
-vm.neon.tech/name: {{ include "vela.autoscaler.name" . }}
-{{- end }}
-
 {{/*
 Create chart name and version as used by the chart label.
 */}}

src/deployment/charts/vela/templates/autoscaler/db-service.yaml

@@ -28,8 +28,6 @@ spec:
       targetPort: 5432
       protocol: TCP
       name: postgres
-  selector:
-{{ include "vela.autoscaler.selectorLabel" . | nindent 4 }}
 ---
 {{- end }}
 apiVersion: v1
@@ -50,5 +48,3 @@ spec:
       targetPort: 5432
       protocol: TCP
       name: postgres
-  selector:
-{{ include "vela.autoscaler.selectorLabel" . | nindent 4 }}

src/deployment/charts/vela/templates/autoscaler/pgbouncer-service.yaml

@@ -13,6 +13,4 @@ spec:
       targetPort: 6432
       protocol: TCP
       name: pgbouncer
-  selector:
-{{ include "vela.autoscaler.selectorLabel" . | nindent 4 }}
 {{- end }}

src/deployment/charts/vela/templates/compose/services.yaml

@@ -13,8 +13,6 @@ spec:
       targetPort: 3000
       protocol: TCP
       name: http
-  selector:
-{{ include "vela.autoscaler.selectorLabel" . | nindent 4 }}
 
 ---
 {{- if .Values.storage.enabled }}
@@ -31,8 +29,6 @@ spec:
       targetPort: 5000
       protocol: TCP
       name: http
-  selector:
-{{ include "vela.autoscaler.selectorLabel" . | nindent 4 }}
 ---
 {{- end }}
 apiVersion: v1
@@ -48,8 +44,6 @@ spec:
       targetPort: 8080
       protocol: TCP
       name: http
-  selector:
-{{ include "vela.autoscaler.selectorLabel" . | nindent 4 }}
 ---
 apiVersion: v1
 kind: Service
@@ -65,6 +59,4 @@ spec:
       targetPort: 9187
       protocol: TCP
       name: http
-  selector:
-{{ include "vela.autoscaler.selectorLabel" . | nindent 4 }}
 {{- end }}

src/deployment/charts/vela/values.yaml

@@ -46,7 +46,7 @@ autoscalerVm:
   fullnameOverride: ""
   powerState: Running
   extraNetwork:
-    enable: false
+    enable: true
   image:
     repository: docker.io/manoharbrm/vela-vm
     tag: latest

src/deployment/kubernetes/__init__.py

@@ -7,7 +7,7 @@
 from kubernetes_asyncio import client
 
 from ...exceptions import VelaKubernetesError
-from ._util import core_v1_client, custom_api_client, storage_v1_client
+from ._util import core_v1_client, custom_api_client, discovery_v1_client, storage_v1_client
 from .neonvm import NeonVM, get_neon_vm
 
 logger = logging.getLogger(__name__)
@@ -143,6 +143,64 @@ async def apply_kong_consumer(self, namespace: str, consumer: dict[str, Any]) ->
                 else:
                     raise
 
+    async def ensure_endpoint_slice(
+        self,
+        namespace: str,
+        slice_name: str,
+        service_name: str,
+        address: str,
+        port: int,
+        port_name: str,
+    ) -> None:
+        body = {
+            "apiVersion": "discovery.k8s.io/v1",
+            "kind": "EndpointSlice",
+            "metadata": {
+                "name": slice_name,
+                "namespace": namespace,
+                "labels": {
+                    "kubernetes.io/service-name": service_name,
+                },
+            },
+            "addressType": "IPv4",
+            "endpoints": [
+                {
+                    "addresses": [address],
+                    "conditions": {"ready": True},
+                }
+            ],
+            "ports": [
+                {
+                    "name": port_name,
+                    "port": port,
+                    "protocol": "TCP",
+                }
+            ],
+        }
+
+        async with discovery_v1_client() as discovery:
+            try:
+                await discovery.create_namespaced_endpoint_slice(namespace, body=body)
+                logger.info(
+                    "Created EndpointSlice %s for service %s/%s via %s",
+                    slice_name,
+                    namespace,
+                    service_name,
+                    address,
+                )
+            except client.exceptions.ApiException as exc:
+                if exc.status == 409:
+                    logger.info(
+                        "EndpointSlice %s already exists for service %s/%s, skipping",
+                        slice_name,
+                        namespace,
+                        service_name,
+                    )
+                    return
+                raise VelaKubernetesError(
+                    f"Failed to create EndpointSlice {slice_name} for {service_name} in {namespace}: {exc.reason}"
+                ) from exc
+
     async def apply_secret(self, namespace: str, secret: dict[str, Any]) -> None:
         name = secret["metadata"]["name"]
         async with core_v1_client() as core_v1:

src/deployment/kubernetes/_util.py

@@ -1,7 +1,7 @@
 from contextlib import asynccontextmanager
 
 from aiohttp import ClientTimeout
-from kubernetes_asyncio.client import ApiClient, CoreV1Api, CustomObjectsApi, StorageV1Api
+from kubernetes_asyncio.client import ApiClient, CoreV1Api, CustomObjectsApi, DiscoveryV1Api, StorageV1Api
 from kubernetes_asyncio.config import load_incluster_config, load_kube_config
 from kubernetes_asyncio.config.config_exception import ConfigException
 
@@ -57,3 +57,9 @@ async def custom_api_client():
 async def storage_v1_client():
     async with api_client() as client:
         yield StorageV1Api(api_client=client)
+
+
+@asynccontextmanager
+async def discovery_v1_client():
+    async with api_client() as client:
+        yield DiscoveryV1Api(api_client=client)