materialized, misc: Version gate new listener config

We create a separate function for resolving the listener config depending on the version / whether there's an upgrade. This is important such that our upgrade tests don't break.

For anything that above that overrides these listener configs, we override these too

Author: SangJunBak

misc/python/materialize/checks/mzcompose_actions.py

@@ -18,6 +18,9 @@
 from materialize.mz_version import MzVersion
 from materialize.mzcompose.helpers.iceberg import setup_polaris_for_iceberg
 from materialize.mzcompose.services.clusterd import Clusterd
+from materialize.mzcompose.services.listener_config import (
+    resolve_listeners_config_path,
+)
 from materialize.mzcompose.services.materialized import DeploymentStatus, Materialized
 from materialize.mzcompose.services.ssh_bastion_host import (
     setup_default_ssh_test_connection,
@@ -86,12 +89,16 @@ def execute(self, e: Executor) -> None:
         )
         print(f"Starting Mz using image {image}, mz_service {self.mz_service}")
 
-        listeners_config_path = (
-            f"{MZ_ROOT}/src/materialized/ci/listener_configs/v26_32_0/testdrive.json"
+        # SASL listeners arrived in v0.158.0, so only newer versions get the
+        # sasl variant.
+        config_name = (
+            "testdrive_sasl"
+            if not self.tag or self.tag >= MzVersion.parse_mz("v0.158.0-dev")
+            else "testdrive"
+        )
+        listeners_config_path = resolve_listeners_config_path(
+            self.tag or MzVersion.parse_cargo(), config_name
         )
-
-        if not self.tag or self.tag >= MzVersion.parse_mz("v0.158.0-dev"):
-            listeners_config_path = f"{MZ_ROOT}/src/materialized/ci/listener_configs/v26_32_0/testdrive_sasl.json"
 
         mz = Materialized(
             name=self.mz_service,

misc/python/materialize/mzcompose/services/listener_config.py

@@ -0,0 +1,31 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+"""Selection of the listener config schema that a Materialize version can parse."""
+
+from materialize import MZ_ROOT
+from materialize.mz_version import MzVersion
+
+
+def resolve_listeners_config_path(
+    image_version: MzVersion,
+    config_name: str = "testdrive",
+) -> str | None:
+    """The `config_name` listeners config to mount for an environmentd version.
+
+    Returns None for an environmentd that predates listener-config support,
+    in which case no config is mounted.
+    """
+    if image_version >= "v26.32.0-dev":
+        return f"{MZ_ROOT}/src/materialized/ci/listener_configs/v26_32_0/{config_name}.json"
+    elif image_version >= "v0.147.0-dev":
+        return f"{MZ_ROOT}/src/materialized/ci/listener_configs/v0_147_0/{config_name}.json"
+    else:
+        # environmentd before v0.147.0 has no MZ_LISTENERS_CONFIG_PATH support.
+        return None

misc/python/materialize/mzcompose/services/materialized.py

@@ -15,7 +15,7 @@
 from enum import Enum
 from typing import Any
 
-from materialize import MZ_ROOT, docker
+from materialize import docker
 from materialize.mz_version import MzVersion
 from materialize.mzcompose import (
     DEFAULT_CRDB_ENVIRONMENT,
@@ -32,6 +32,9 @@
 )
 from materialize.mzcompose.services import foundationdb
 from materialize.mzcompose.services.azurite import azure_blob_uri
+from materialize.mzcompose.services.listener_config import (
+    resolve_listeners_config_path,
+)
 from materialize.mzcompose.services.metadata_store import (
     EXTERNAL_METADATA_STORE_ADDRESS,
     METADATA_STORE,
@@ -102,7 +105,7 @@ def __init__(
         default_replication_factor: int = 1,
         builtin_system_cluster_replication_factor: int | None = None,
         builtin_probe_cluster_replication_factor: int | None = None,
-        listeners_config_path: str = f"{MZ_ROOT}/src/materialized/ci/listener_configs/testdrive.json",
+        listeners_config_path: str | None = None,
         config_sync_file_path: str | None = None,
         support_external_clusterd: bool = False,
         networks: (
@@ -340,9 +343,12 @@ def __init__(
         if platform:
             config["platform"] = platform
 
-        if image_version is None or image_version >= "v0.147.0-dev":
-            assert os.path.exists(listeners_config_path)
-            volumes.append(f"{listeners_config_path}:/listeners_config")
+        config_path = listeners_config_path or resolve_listeners_config_path(
+            image_version or MzVersion.parse_cargo()
+        )
+        if config_path is not None:
+            assert os.path.exists(config_path)
+            volumes.append(f"{config_path}:/listeners_config")
             environment.append("MZ_LISTENERS_CONFIG_PATH=/listeners_config")
 
         if config_sync_file_path is not None:

misc/python/materialize/scalability/endpoint/endpoints.py

@@ -9,9 +9,14 @@
 
 import psycopg
 
-from materialize import MZ_ROOT, git
+from materialize import git
+from materialize.docker import get_mz_version_from_image_tag
+from materialize.mz_version import MzVersion
 from materialize.mzcompose import ADDITIONAL_BENCHMARKING_SYSTEM_PARAMETERS
 from materialize.mzcompose.composition import Composition
+from materialize.mzcompose.services.listener_config import (
+    resolve_listeners_config_path,
+)
 from materialize.mzcompose.services.materialized import Materialized
 from materialize.mzcompose.services.postgres import Postgres
 from materialize.scalability.endpoint.endpoint import ConnectionKind, Endpoint
@@ -180,12 +185,22 @@ def up(self) -> None:
         self.lift_limits()
 
     def up_internal(self) -> None:
+        # Match the schema dir to the image's version, since an older
+        # environmentd cannot parse a newer listener config schema.
+        image_version = (
+            MzVersion.parse_cargo()
+            if self.image is None
+            else get_mz_version_from_image_tag(self.image.rsplit(":", 1)[1])
+        )
+        listeners_config_path = resolve_listeners_config_path(
+            image_version, "testdrive_sasl"
+        )
         with self.composition.override(
             Materialized(
                 image=self.image,
                 sanity_restart=False,
                 additional_system_parameter_defaults=ADDITIONAL_BENCHMARKING_SYSTEM_PARAMETERS,
-                listeners_config_path=f"{MZ_ROOT}/src/materialized/ci/listener_configs/v26_32_0/testdrive_sasl.json",
+                listeners_config_path=listeners_config_path,
             )
         ):
             self.composition.up("materialized")