From 6048ac5f38bc870052a54dc07da1670f04ae9e93 Mon Sep 17 00:00:00 2001 From: wind57 Date: Sun, 3 May 2026 22:24:39 +0300 Subject: [PATCH 1/5] first test run Signed-off-by: wind57 --- .../fabric8/client/reload/TestAssertions.java | 2 +- .../integration/tests/commons/Commons.java | 106 ++++++++---------- .../integration/tests/commons/Constants.java | 7 -- .../tests/commons/FixedPortsK3sContainer.java | 91 +++++++++++++-- .../Fabric8ClientKubernetesFixture.java | 51 ++++++--- .../FabricClientIntegrationTestExtension.java | 4 +- .../NativeClientIntegrationTestExtension.java | 2 +- .../NativeClientKubernetesFixture.java | 33 ++++-- 8 files changed, 195 insertions(+), 101 deletions(-) diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/reload/TestAssertions.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/reload/TestAssertions.java index 2280049998..5ec746884c 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/reload/TestAssertions.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/reload/TestAssertions.java @@ -96,7 +96,7 @@ static void manifests(Phase phase, Fabric8ClientKubernetesFixture fabric8Kuberne if (phase.equals(Phase.CREATE)) { fabric8KubernetesFixture.createAndWait(namespace, configMap, null); - fabric8KubernetesFixture.createAndWait(namespace, null, deployment, service, true); + fabric8KubernetesFixture.createAndWait(namespace, deployment, service, true); } else { fabric8KubernetesFixture.deleteAndWait(namespace, configMap, null); diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java index 476aca831a..a4f05db7de 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java @@ -17,14 +17,9 @@ package org.springframework.cloud.kubernetes.integration.tests.commons; import java.io.File; -import java.io.IOException; import java.io.InputStream; -import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.time.Duration; import java.util.Arrays; import java.util.List; @@ -33,7 +28,6 @@ import com.github.dockerjava.api.command.ListImagesCmd; import com.github.dockerjava.api.command.PullImageCmd; -import com.github.dockerjava.api.command.SaveImageCmd; import com.github.dockerjava.api.model.Image; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -50,9 +44,9 @@ import org.springframework.web.reactive.function.client.WebClient; import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.KUBERNETES_VERSION_FILE; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TEMP_FOLDER; import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TMP_IMAGES; import static org.springframework.cloud.kubernetes.integration.tests.commons.FixedPortsK3sContainer.CONTAINER; +import static org.springframework.cloud.kubernetes.integration.tests.commons.FixedPortsK3sContainer.REGISTRY_PORT; /** * A few commons things that can be re-used across clients. This is meant to be used for @@ -78,9 +72,10 @@ public static K3sContainer container() { return CONTAINER; } - public static void loadSpringCloudKubernetesImage(String project, K3sContainer container) { + public static void tagAndPushSpringCloudKubernetesImage(String imageName, K3sContainer container) { try { - loadImage("springcloud/" + project, pomVersion(), project, container); + String springCloudImage = "springcloud/" + imageName + ":" + pomVersion(); + tagAndPushImage(springCloudImage, container); } catch (Exception e) { throw new RuntimeException(e); @@ -88,56 +83,53 @@ public static void loadSpringCloudKubernetesImage(String project, K3sContainer c } /** - * create a tar, copy it in the running k3s and load this tar as an image. + * tag image and push to local registry. */ - public static void loadImage(String image, String tag, String tarName, K3sContainer container) { + public static void tagAndPushImage(String imageFromDeploymentWithTag, K3sContainer container) { - if (imageAlreadyInK3s(container, tarName)) { + if (imageAlreadyInK3s(container, imageFromDeploymentWithTag)) { return; } - // save image - try (SaveImageCmd saveImageCmd = container.getDockerClient().saveImageCmd(image)) { - InputStream imageStream = saveImageCmd.withTag(tag).exec(); - - Path imagePath = Paths.get(TEMP_FOLDER + "/" + tarName + ".tar"); - try { - Files.copy(imageStream, imagePath, StandardCopyOption.REPLACE_EXISTING); - } - catch (IOException e) { - throw new UncheckedIOException(e); + try { + int lastColon = imageFromDeploymentWithTag.lastIndexOf(':'); + if (lastColon < 0) { + throw new IllegalArgumentException("image must include tag: " + imageFromDeploymentWithTag); } - // import image with ctr. this works because TEMP_FOLDER is mounted in the - // container + + String imageWithoutTag = imageFromDeploymentWithTag.substring(0, lastColon); + String tag = imageFromDeploymentWithTag.substring(lastColon + 1); + + String targetRepository = "localhost:" + REGISTRY_PORT + "/" + imageWithoutTag; + String targetImageWithTag = targetRepository + ":" + tag; + + container.getDockerClient().tagImageCmd(imageFromDeploymentWithTag, targetRepository, tag).exec(); + Awaitilities.awaitUntil(120, 1000, () -> { - Container.ExecResult result; try { - result = container.execInContainer("ctr", "i", "import", - Constants.TEMP_FOLDER + "/" + tarName + ".tar"); + container.getDockerClient().pushImageCmd(targetImageWithTag).start().awaitCompletion(); + return true; } catch (Exception e) { - throw new RuntimeException(e); + LOG.info("failed to push image " + targetImageWithTag + " to local registry", e); + return false; } - boolean noErrors = result.getStderr() == null || result.getStderr().isEmpty(); - if (!noErrors) { - LOG.info("error is : " + result.getStderr()); - } - return noErrors; }); } - + catch (Exception e) { + throw new RuntimeException(e); + } } /** - * Ensures a common external test image is available inside K3s/containerd. - * It first checks whether the image is already present in K3s. - * If not, it tries to load it as a tar under '/tmp/docker/images'. - * If no matching tar is found, it pulls the image directly inside K3s - * using 'ctr images pull'. - * This is meant for shared test images such as busybox, wiremock, kafka, etc. + * Ensures a common external test image is available inside K3s/containerd. It first + * checks whether the image is already present in K3s. If not, it tries to load it as + * a tar under '/tmp/docker/images'. If no matching tar is found, it pulls the image + * directly inside K3s using 'ctr images pull'. This is meant for shared test images + * such as busybox, wiremock, kafka, etc. */ - public static void loadOrPullCommonTestImages(K3sContainer container, String tarName, - String imageNameForDownload, String imageVersion) { + public static void loadOrPullCommonTestImages(K3sContainer container, String tarName, String imageNameForDownload, + String imageVersion) { if (imageAlreadyInK3s(container, tarName)) { return; @@ -239,15 +231,14 @@ public static void validateImage(String image, K3sContainer container) { } } - public static void pullImage(String image, String tag, String tarName, K3sContainer container) - throws InterruptedException { + public static void pullImage(String imageFromDeployment, K3sContainer container) throws InterruptedException { - if (imageAlreadyInK3s(container, tarName)) { + if (imageAlreadyInK3s(container, imageFromDeployment)) { return; } - try (PullImageCmd pullImageCmd = container.getDockerClient().pullImageCmd(image)) { - pullImageCmd.withTag(tag).start().awaitCompletion(); + try (PullImageCmd pullImageCmd = container.getDockerClient().pullImageCmd(imageFromDeployment)) { + pullImageCmd.start().awaitCompletion(); } } @@ -324,24 +315,19 @@ private static void loadImageFromPath(String tarName, K3sContainer container) { }); } - private static boolean imageAlreadyInK3s(K3sContainer container, String tarName) { + private static boolean imageAlreadyInK3s(K3sContainer container, String imageWithTag) { + try { + String stdout = container.execInContainer("ctr", "-n", "k8s.io", "images", "list", "-q").getStdout(); - if (tarName == null) { - return false; - } + boolean present = Arrays.stream(stdout.split("\\R")).map(String::trim).anyMatch(imageWithTag::equals); - try { - boolean present = container.execInContainer("sh", "-c", "ctr images list | grep " + tarName) - .getStdout() - .contains(tarName); if (present) { - System.out.println("image : " + tarName + " already in k3s, skipping"); + System.out.println("image : " + imageWithTag + " already in k3s, skipping"); return true; } - else { - System.out.println("image : " + tarName + " not in k3s"); - return false; - } + + System.out.println("image : " + imageWithTag + " not in k3s"); + return false; } catch (Exception e) { throw new RuntimeException(e); diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Constants.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Constants.java index 811299300b..c3c895667f 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Constants.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Constants.java @@ -16,8 +16,6 @@ package org.springframework.cloud.kubernetes.integration.tests.commons; -import java.io.File; - /** * @author wind57 */ @@ -32,11 +30,6 @@ private Constants() { */ static final String TMP_IMAGES = "/tmp/docker/images"; - /** - * Temporary folder where to load images. - */ - static final String TEMP_FOLDER = new File(System.getProperty("java.io.tmpdir")).getAbsolutePath(); - /** * where is the version situated. */ diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java index 92315d8dfb..94f5dd42d1 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java @@ -16,14 +16,22 @@ package org.springframework.cloud.kubernetes.integration.tests.commons; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Objects; import com.github.dockerjava.api.model.Bind; import com.github.dockerjava.api.model.HostConfig; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; import org.testcontainers.k3s.K3sContainer; import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.MountableFile; + +import org.springframework.test.util.TestSocketUtils; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TEMP_FOLDER; import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TMP_IMAGES; /** @@ -32,49 +40,114 @@ * * @author wind57 */ -final class FixedPortsK3sContainer extends K3sContainer { +public final class FixedPortsK3sContainer extends K3sContainer { /** * Test containers exposed ports. */ private static final int[] EXPOSED_PORTS = new int[] { 80, 6443, 8080, 8888, 9092, 32321, 32322 }; + /** + * Port for the local running images registry. + */ + public static final int REGISTRY_PORT = TestSocketUtils.findAvailableTcpPort(); + + private static final Network NETWORK = Network.newNetwork(); + + private static final Path REGISTRIES_YAML = createRegistriesYaml(); + + private static final LocalRegistryContainer REGISTRY = new LocalRegistryContainer().configureFixedPorts() + .withNetwork(NETWORK) + .withNetworkAliases("registry") + .withReuse(true); + /** * Rancher version to use for test-containers. */ private static final String RANCHER_VERSION = "rancher/k3s:v1.34.1-k3s1"; + /** + * Docker registry version. + */ + private static final String DOCKER_REGISTRY_VERSION = "registry:3.1.0"; + /** * Command to use when starting rancher. Without "server" option, traefik is not * installed */ - private static final String RANCHER_COMMAND = "server --disable=metric-server"; + private static final String RANCHER_COMMAND = "server --disable=metric-server --disable-default-registry-endpoint"; static final K3sContainer CONTAINER = new FixedPortsK3sContainer(DockerImageName.parse(RANCHER_VERSION)) .configureFixedPorts() .addBinds() .withCommand(RANCHER_COMMAND) - .withReuse(true); + .withReuse(true) + .withNetwork(NETWORK) + .withCopyFileToContainer(MountableFile.forHostPath(REGISTRIES_YAML), "/etc/rancher/k3s/registries.yaml"); - FixedPortsK3sContainer(DockerImageName dockerImageName) { + private FixedPortsK3sContainer(DockerImageName dockerImageName) { super(dockerImageName); + REGISTRY.start(); } - FixedPortsK3sContainer configureFixedPorts() { + private FixedPortsK3sContainer configureFixedPorts() { for (int port : EXPOSED_PORTS) { super.addFixedExposedPort(port, port); } return this; } - FixedPortsK3sContainer addBinds() { + private FixedPortsK3sContainer addBinds() { super.withCreateContainerCmdModifier(cmd -> { HostConfig hostConfig = Objects.requireNonNull(cmd.getHostConfig()); - hostConfig.withBinds(Bind.parse(TEMP_FOLDER + ":" + TEMP_FOLDER), - Bind.parse(TMP_IMAGES + ":" + TMP_IMAGES)); + hostConfig.withBinds(Bind.parse(TMP_IMAGES + ":" + TMP_IMAGES)); }); return this; } + /** + * Small doc on how the set-up works. ( 5000 is just an example )
+	 *     - we start a local registry and expose it on localhost:<5000>
+	 *     - from the host, we can push an image with:
+	 *       docker push localhost:5000/image:tag
+	 *     - the image is now stored in that local registry
+	 *     - k3s later sees the image reference: localhost:5000/image:tag
+	 *     - inside the k3s container, localhost would point to k3s itself, not to the registry
+	 *     - because of the mirror entry in registries.yaml, when k3s/containerd sees
+	 *       an image starting with localhost:5000, it actually uses the endpoint 'http://registry:5000'
+	 *     - LocalRegistryContainer has withNetworkAliases("registry")
+	 *     - this makes the registry reachable from the same Docker network via the hostname "registry"
+	 * 
+ */ + private static Path createRegistriesYaml() { + try { + Path file = Files.createTempFile("registries", ".yaml"); + // can't use text blocks here because of checkstyle + String content = "mirrors:\n" + " \"localhost:%s\":\n" + " endpoint:\n" + + " - \"http://registry:5000\"\n"; + Files.writeString(file, content.formatted(REGISTRY_PORT)); + return file; + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * official registry image with fixed port 5000. + */ + private static final class LocalRegistryContainer extends GenericContainer { + + private LocalRegistryContainer() { + super(DOCKER_REGISTRY_VERSION); + } + + private LocalRegistryContainer configureFixedPorts() { + addFixedExposedPort(REGISTRY_PORT, 5000); + return this; + } + + } + } diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Fabric8ClientKubernetesFixture.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Fabric8ClientKubernetesFixture.java index ab5859a25a..545d342109 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Fabric8ClientKubernetesFixture.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Fabric8ClientKubernetesFixture.java @@ -44,9 +44,10 @@ import org.springframework.cloud.kubernetes.integration.tests.commons.Images; import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.loadImage; import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.pomVersion; import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.pullImage; +import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.tagAndPushImage; +import static org.springframework.cloud.kubernetes.integration.tests.commons.FixedPortsK3sContainer.REGISTRY_PORT; /** * @author wind57 @@ -70,31 +71,41 @@ public Fabric8ClientKubernetesFixture(K3sContainer container) { * service. It creates the given resources as-well as waits for them to be created. * The delay check is intentionally not taken as an argument, so that it stays as * tight as possible, providing reasonable defaults. - * + * @param imageWithoutTag is true when we need to set the correct pom version, as such + * it only relates to images from our own code ( not busybox, wiremock, etc. ) */ - public void createAndWait(String namespace, String name, @Nullable Deployment deployment, @Nullable Service service, - boolean changeVersion) { + public void createAndWait(String namespace, @Nullable Deployment deployment, @Nullable Service service, + boolean imageWithoutTag) { try { if (deployment != null) { + String imageFromDeployment = deployment.getSpec() .getTemplate() .getSpec() .getContainers() .get(0) .getImage(); - if (changeVersion) { + + if (imageWithoutTag) { + + String imageFromDeploymentWithTag = imageFromDeployment + ":" + pomVersion(); + + // change format to localhost:5000/.... + String imageFormatForRegistry = springCloudImageInLocalRegistry(imageFromDeploymentWithTag); + deployment.getSpec() .getTemplate() .getSpec() .getContainers() .get(0) - .setImage(imageFromDeployment + ":" + pomVersion()); + .setImage(imageFormatForRegistry); } else { - String[] image = imageFromDeployment.split(":", 2); - pullImage(image[0], image[1], name, container); - loadImage(image[0], image[1], name, container); + // pullImage is only needed when we run some test locally. + // Inside github actions, this will be a NOOP. + pullImage(imageFromDeployment, container); + tagAndPushImage(imageFromDeployment, container); } client.apps().deployments().inNamespace(namespace).resource(deployment).create(); @@ -123,7 +134,7 @@ public void busybox(String namespace, Phase phase) { Service service = Serialization.unmarshal(serviceStream, Service.class); if (phase.equals(Phase.CREATE)) { - createAndWait(namespace, "busybox", deployment, service, false); + createAndWait(namespace, deployment, service, false); } else if (phase.equals(Phase.DELETE)) { deleteAndWait(namespace, deployment, service); @@ -134,7 +145,7 @@ public void externalName(Phase phase) { InputStream serviceStream = inputStream("external-name-service/external-name-service.yaml"); Service service = Serialization.unmarshal(serviceStream, Service.class); if (Phase.CREATE.equals(phase)) { - createAndWait("default", null, null, service, false); + createAndWait("default", null, service, false); } else { deleteAndWait("default", null, service); @@ -288,7 +299,7 @@ public void istioCtl(String namespace, Phase phase) { istioctlDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).setImage(imageWithVersion); if (phase.equals(Phase.CREATE)) { - createAndWait(namespace, null, istioctlDeployment, null, false); + createAndWait(namespace, istioctlDeployment, null, false); } else { deleteAndWait(namespace, istioctlDeployment, null); @@ -351,7 +362,7 @@ public void wiremock(String namespace, Phase phase, boolean withNodePort) { if (phase.equals(Phase.CREATE)) { deployment.getMetadata().setNamespace(namespace); service.getMetadata().setNamespace(namespace); - createAndWait(namespace, "wiremock", deployment, service, false); + createAndWait(namespace, deployment, service, false); } else { deleteAndWait(namespace, deployment, service); @@ -368,7 +379,7 @@ public void configWatcher(Phase phase) { Service service = client.services().load(serviceStream).item(); if (phase.equals(Phase.CREATE)) { - createAndWait("default", deployment.getMetadata().getName(), deployment, service, true); + createAndWait("default", deployment, service, true); } else if (phase.equals(Phase.DELETE)) { deleteAndWait("default", deployment, service); @@ -465,6 +476,18 @@ private void innerSetup(String namespace, InputStream serviceAccountAsStream, In } } + // from docker.io/springcloud/spring-cloud-kubernetes-configuration-watcher -> + // localhost:5000/springcloud/spring-cloud-kubernetes-configuration-watcher:5.0.2-SNAPSHOT + private static String springCloudImageInLocalRegistry(String imageFromDeploymentWithTag) { + String image = imageFromDeploymentWithTag; + + if (image.startsWith("docker.io/")) { + image = image.substring("docker.io/".length()); + } + + return "localhost:" + REGISTRY_PORT + "/" + image; + } + private String deploymentName(Deployment deployment) { return deployment.getMetadata().getName(); } diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/FabricClientIntegrationTestExtension.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/FabricClientIntegrationTestExtension.java index 6c805a193a..eec839900b 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/FabricClientIntegrationTestExtension.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/FabricClientIntegrationTestExtension.java @@ -66,7 +66,7 @@ public void beforeAll(ExtensionContext context) { // 2. external image presence for (String image : scenario.withImages()) { Commons.validateImage(image, container); - Commons.loadSpringCloudKubernetesImage(image, container); + Commons.tagAndPushSpringCloudKubernetesImage(image, container); } // 3. deploy istio @@ -230,7 +230,7 @@ private void istioSetup(K3sContainer container, Fabric8ClientKubernetesFixture f Deployment deployment = Serialization.unmarshal(deploymentStream, Deployment.class); Service service = Serialization.unmarshal(serviceStream, Service.class); - fabric8KubernetesFixture.createAndWait("istio-test", null, deployment, service, true); + fabric8KubernetesFixture.createAndWait("istio-test", deployment, service, true); } catch (Exception e) { diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java index 5af4da9ee4..e93a558af2 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java @@ -60,7 +60,7 @@ public void beforeAll(ExtensionContext context) throws Exception { // 2. external image presence for (String image : scenario.withImages()) { Commons.validateImage(image, container); - Commons.loadSpringCloudKubernetesImage(image, container); + Commons.tagAndPushSpringCloudKubernetesImage(image, container); } // 3. set-up RBAC. diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java index 4b567c8f1d..e9220e0c2f 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java @@ -61,9 +61,10 @@ import org.springframework.cloud.kubernetes.integration.tests.commons.Images; import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.loadImage; import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.pomVersion; import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.pullImage; +import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.tagAndPushImage; +import static org.springframework.cloud.kubernetes.integration.tests.commons.FixedPortsK3sContainer.REGISTRY_PORT; /** * @author wind57 @@ -107,7 +108,7 @@ public NativeClientKubernetesFixture(K3sContainer container) { * */ public void createAndWait(String namespace, String name, V1Deployment deployment, V1Service service, - boolean changeVersion) { + boolean imageWithoutTag) { try { coreV1Api.createNamespacedService(namespace, service).execute(); @@ -119,18 +120,24 @@ public void createAndWait(String namespace, String name, V1Deployment deployment .getContainers() .get(0) .getImage(); - if (changeVersion) { + + if (imageWithoutTag) { + + String imageFromDeploymentWithTag = imageFromDeployment + ":" + pomVersion(); + + // change format to localhost:5000/.... + String imageFormatForRegistry = springCloudImageInLocalRegistry(imageFromDeploymentWithTag); + deployment.getSpec() .getTemplate() .getSpec() .getContainers() .get(0) - .setImage(imageFromDeployment + ":" + pomVersion()); + .setImage(imageFormatForRegistry); } else { - String[] image = imageFromDeployment.split(":", 2); - pullImage(image[0], image[1], name, container); - loadImage(image[0], image[1], name, container); + pullImage(imageFromDeployment, container); + tagAndPushImage(imageFromDeployment, container); } appsV1Api.createNamespacedDeployment(namespace, deployment).execute(); @@ -789,6 +796,18 @@ private String labelSelector(Map labels) { return labels.entrySet().stream().map(en -> en.getKey() + "=" + en.getValue()).collect(Collectors.joining(",")); } + // from docker.io/springcloud/spring-cloud-kubernetes-configuration-watcher -> + // localhost:5000/springcloud/spring-cloud-kubernetes-configuration-watcher:5.0.2-SNAPSHOT + private static String springCloudImageInLocalRegistry(String imageFromDeploymentWithTag) { + String image = imageFromDeploymentWithTag; + + if (image.startsWith("docker.io/")) { + image = image.substring("docker.io/".length()); + } + + return "localhost:" + REGISTRY_PORT + "/" + image; + } + private interface CheckedSupplier { T get() throws Exception; From 66ab297cff13818938553bfb834ab2e49ac99c56 Mon Sep 17 00:00:00 2001 From: wind57 Date: Mon, 4 May 2026 16:13:58 +0300 Subject: [PATCH 2/5] another test run Signed-off-by: wind57 --- .../integration/tests/commons/Commons.java | 31 +++++----- .../tests/commons/FixedIdNetworkProvider.java | 58 +++++++++++++++++++ .../tests/commons/FixedPortsK3sContainer.java | 58 ++++++------------- .../integration/tests/commons/Images.java | 8 ++- .../FabricClientIntegrationTestExtension.java | 2 +- .../NativeClientIntegrationTestExtension.java | 2 +- .../src/main/resources/registries.yaml | 4 ++ 7 files changed, 106 insertions(+), 57 deletions(-) create mode 100644 spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedIdNetworkProvider.java create mode 100644 spring-cloud-kubernetes-test-support/src/main/resources/registries.yaml diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java index a4f05db7de..0583ef1179 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java @@ -72,10 +72,11 @@ public static K3sContainer container() { return CONTAINER; } - public static void tagAndPushSpringCloudKubernetesImage(String imageName, K3sContainer container) { + public static void tagAndPushSpringCloudKubernetesImageToLocalDockerRegistry(String imageNameWithoutTag, + K3sContainer container) { try { - String springCloudImage = "springcloud/" + imageName + ":" + pomVersion(); - tagAndPushImage(springCloudImage, container); + String springCloudImageWithTag = "springcloud/" + imageNameWithoutTag + ":" + pomVersion(); + tagAndPushImage(springCloudImageWithTag, container); } catch (Exception e) { throw new RuntimeException(e); @@ -85,25 +86,25 @@ public static void tagAndPushSpringCloudKubernetesImage(String imageName, K3sCon /** * tag image and push to local registry. */ - public static void tagAndPushImage(String imageFromDeploymentWithTag, K3sContainer container) { + public static void tagAndPushImage(String imageNameWithTag, K3sContainer container) { - if (imageAlreadyInK3s(container, imageFromDeploymentWithTag)) { + if (imageAlreadyInK3s(container, imageNameWithTag)) { return; } try { - int lastColon = imageFromDeploymentWithTag.lastIndexOf(':'); + int lastColon = imageNameWithTag.lastIndexOf(':'); if (lastColon < 0) { - throw new IllegalArgumentException("image must include tag: " + imageFromDeploymentWithTag); + throw new IllegalArgumentException("image must include tag: " + imageNameWithTag); } - String imageWithoutTag = imageFromDeploymentWithTag.substring(0, lastColon); - String tag = imageFromDeploymentWithTag.substring(lastColon + 1); + String imageWithoutTag = imageNameWithTag.substring(0, lastColon); + String tag = imageNameWithTag.substring(lastColon + 1); String targetRepository = "localhost:" + REGISTRY_PORT + "/" + imageWithoutTag; String targetImageWithTag = targetRepository + ":" + tag; - container.getDockerClient().tagImageCmd(imageFromDeploymentWithTag, targetRepository, tag).exec(); + container.getDockerClient().tagImageCmd(imageNameWithTag, targetRepository, tag).exec(); Awaitilities.awaitUntil(120, 1000, () -> { try { @@ -217,7 +218,7 @@ private static String fullImageReference(String imageName, String imageVersion) } /** - * validates that the provided image does exist in the local docker registry. + * validates that the provided image does exist in the local dcoker cache. */ public static void validateImage(String image, K3sContainer container) { try (ListImagesCmd listImagesCmd = container.getDockerClient().listImagesCmd()) { @@ -319,14 +320,16 @@ private static boolean imageAlreadyInK3s(K3sContainer container, String imageWit try { String stdout = container.execInContainer("ctr", "-n", "k8s.io", "images", "list", "-q").getStdout(); - boolean present = Arrays.stream(stdout.split("\\R")).map(String::trim).anyMatch(imageWithTag::equals); + boolean present = Arrays.stream(stdout.split("\\R")) + .map(String::trim) + .anyMatch(line -> line.contains(imageWithTag)); if (present) { - System.out.println("image : " + imageWithTag + " already in k3s, skipping"); + LOG.info("image : " + imageWithTag + " already in k3s, skipping"); return true; } - System.out.println("image : " + imageWithTag + " not in k3s"); + LOG.info("image : " + imageWithTag + " not in k3s"); return false; } catch (Exception e) { diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedIdNetworkProvider.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedIdNetworkProvider.java new file mode 100644 index 0000000000..8c40d131cc --- /dev/null +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedIdNetworkProvider.java @@ -0,0 +1,58 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.integration.tests.commons; + +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.Network; + +/** + * re-use implementation for a Network. It first checks if such a named network already + * exists via docker client API and creates a new one if it does not. + * + * @author wind57 + */ +final class FixedIdNetworkProvider { + + private FixedIdNetworkProvider() { + + } + + static Network createReusableNetwork(String name) { + var client = DockerClientFactory.instance().client(); + + String id = client.listNetworksCmd() + .exec() + .stream() + .filter(network -> name.equals(network.getName())) + .map(com.github.dockerjava.api.model.Network::getId) + .findFirst() + .orElseGet(() -> client.createNetworkCmd().withName(name).withCheckDuplicate(true).exec().getId()); + + return new org.testcontainers.containers.Network() { + @Override + public String getId() { + return id; + } + + @Override + public void close() { + // intentionally no-op + } + }; + } + +} diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java index 94f5dd42d1..a9b2fb249c 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java @@ -16,10 +16,6 @@ package org.springframework.cloud.kubernetes.integration.tests.commons; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Objects; import com.github.dockerjava.api.model.Bind; @@ -30,9 +26,8 @@ import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.MountableFile; -import org.springframework.test.util.TestSocketUtils; - import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TMP_IMAGES; +import static org.springframework.cloud.kubernetes.integration.tests.commons.FixedIdNetworkProvider.createReusableNetwork; /** * A K3sContainer, but with fixed port mappings. This is needed because of the nature of @@ -50,11 +45,23 @@ public final class FixedPortsK3sContainer extends K3sContainer { /** * Port for the local running images registry. */ - public static final int REGISTRY_PORT = TestSocketUtils.findAvailableTcpPort(); - - private static final Network NETWORK = Network.newNetwork(); + public static final int REGISTRY_PORT = 6000; - private static final Path REGISTRIES_YAML = createRegistriesYaml(); + /** + * Small doc on how the set-up works. ( 5000 is just an example )
+	 *     - we start a local registry and expose it on localhost:<5000>
+	 *     - from the host, we can push an image with:
+	 *       docker push localhost:5000/image:tag
+	 *     - the image is now stored in that local registry
+	 *     - k3s later sees the image reference: localhost:5000/image:tag
+	 *     - inside the k3s container, localhost would point to k3s itself, not to the registry
+	 *     - because of the mirror entry in registries.yaml, when k3s/containerd sees
+	 *       an image starting with localhost:5000, it actually uses the endpoint 'http://registry:5000'
+	 *     - LocalRegistryContainer has withNetworkAliases("registry")
+	 *     - this makes the registry reachable from the same Docker network via the hostname "registry"
+	 * 
+ */ + private static final Network NETWORK = createReusableNetwork("spring-cloud-kubernetes-local-docker-registry"); private static final LocalRegistryContainer REGISTRY = new LocalRegistryContainer().configureFixedPorts() .withNetwork(NETWORK) @@ -83,7 +90,8 @@ public final class FixedPortsK3sContainer extends K3sContainer { .withCommand(RANCHER_COMMAND) .withReuse(true) .withNetwork(NETWORK) - .withCopyFileToContainer(MountableFile.forHostPath(REGISTRIES_YAML), "/etc/rancher/k3s/registries.yaml"); + .withCopyFileToContainer(MountableFile.forClasspathResource("registries.yaml"), + "/etc/rancher/k3s/registries.yaml"); private FixedPortsK3sContainer(DockerImageName dockerImageName) { super(dockerImageName); @@ -106,34 +114,6 @@ private FixedPortsK3sContainer addBinds() { return this; } - /** - * Small doc on how the set-up works. ( 5000 is just an example )
-	 *     - we start a local registry and expose it on localhost:<5000>
-	 *     - from the host, we can push an image with:
-	 *       docker push localhost:5000/image:tag
-	 *     - the image is now stored in that local registry
-	 *     - k3s later sees the image reference: localhost:5000/image:tag
-	 *     - inside the k3s container, localhost would point to k3s itself, not to the registry
-	 *     - because of the mirror entry in registries.yaml, when k3s/containerd sees
-	 *       an image starting with localhost:5000, it actually uses the endpoint 'http://registry:5000'
-	 *     - LocalRegistryContainer has withNetworkAliases("registry")
-	 *     - this makes the registry reachable from the same Docker network via the hostname "registry"
-	 * 
- */ - private static Path createRegistriesYaml() { - try { - Path file = Files.createTempFile("registries", ".yaml"); - // can't use text blocks here because of checkstyle - String content = "mirrors:\n" + " \"localhost:%s\":\n" + " endpoint:\n" - + " - \"http://registry:5000\"\n"; - Files.writeString(file, content.formatted(REGISTRY_PORT)); - return file; - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - /** * official registry image with fixed port 5000. */ diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Images.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Images.java index ef76b8009f..ea23b64c2e 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Images.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Images.java @@ -19,6 +19,8 @@ import java.io.BufferedReader; import java.io.InputStreamReader; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.testcontainers.k3s.K3sContainer; /** @@ -26,6 +28,8 @@ */ public final class Images { + private static final Log LOG = LogFactory.getLog(Images.class); + private static final String BUSYBOX = "busybox"; private static final String BUSYBOX_TAR = BUSYBOX + ":" + busyboxVersion(); @@ -116,11 +120,11 @@ private static boolean imageAlreadyInK3s(K3sContainer container, String tarName) .getStdout() .contains(tarName); if (present) { - System.out.println("image : " + tarName + " already in k3s, skipping"); + LOG.info("image : " + tarName + " already in k3s, skipping"); return true; } else { - System.out.println("image : " + tarName + " not in k3s"); + LOG.info("image : " + tarName + " not in k3s"); return false; } } diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/FabricClientIntegrationTestExtension.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/FabricClientIntegrationTestExtension.java index eec839900b..0aea0c91d6 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/FabricClientIntegrationTestExtension.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/FabricClientIntegrationTestExtension.java @@ -66,7 +66,7 @@ public void beforeAll(ExtensionContext context) { // 2. external image presence for (String image : scenario.withImages()) { Commons.validateImage(image, container); - Commons.tagAndPushSpringCloudKubernetesImage(image, container); + Commons.tagAndPushSpringCloudKubernetesImageToLocalDockerRegistry(image, container); } // 3. deploy istio diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java index e93a558af2..fdf8449d89 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java @@ -60,7 +60,7 @@ public void beforeAll(ExtensionContext context) throws Exception { // 2. external image presence for (String image : scenario.withImages()) { Commons.validateImage(image, container); - Commons.tagAndPushSpringCloudKubernetesImage(image, container); + Commons.tagAndPushSpringCloudKubernetesImageToLocalDockerRegistry(image, container); } // 3. set-up RBAC. diff --git a/spring-cloud-kubernetes-test-support/src/main/resources/registries.yaml b/spring-cloud-kubernetes-test-support/src/main/resources/registries.yaml new file mode 100644 index 0000000000..f8539cad31 --- /dev/null +++ b/spring-cloud-kubernetes-test-support/src/main/resources/registries.yaml @@ -0,0 +1,4 @@ +mirrors: + "localhost:6000": + endpoint: + - "http://registry:5000" From fc5b59ddd961e7968891bdfdca0e0e122d84e54d Mon Sep 17 00:00:00 2001 From: wind57 Date: Mon, 4 May 2026 16:41:53 +0300 Subject: [PATCH 3/5] add loading Signed-off-by: wind57 --- .../tests/commons/k3s/NativeClientIntegrationTestExtension.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java index fdf8449d89..b1183b0fa2 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java @@ -110,11 +110,13 @@ public void beforeAll(ExtensionContext context) throws Exception { // 10. deploy kafka if (scenario.deployKafka()) { + Images.loadKafka(container); nativeClientKubernetesFixture.kafka(Phase.CREATE); } // 11. deploy rabbitMq if (scenario.deployRabbitMq()) { + Images.loadRabbitmq(container); nativeClientKubernetesFixture.rabbitMq(Phase.CREATE); } } From e1c2354ea91bf46e673aaa5366cd9ef46dde5e00 Mon Sep 17 00:00:00 2001 From: wind57 Date: Mon, 4 May 2026 19:09:22 +0300 Subject: [PATCH 4/5] trigger Signed-off-by: wind57 From b064705887aa086c4eaacece438bfd8d3cad40db Mon Sep 17 00:00:00 2001 From: wind57 Date: Mon, 4 May 2026 20:27:57 +0300 Subject: [PATCH 5/5] trigger Signed-off-by: wind57