diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/reload/TestAssertions.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/reload/TestAssertions.java index 2280049998..5ec746884c 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/reload/TestAssertions.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/reload/TestAssertions.java @@ -96,7 +96,7 @@ static void manifests(Phase phase, Fabric8ClientKubernetesFixture fabric8Kuberne if (phase.equals(Phase.CREATE)) { fabric8KubernetesFixture.createAndWait(namespace, configMap, null); - fabric8KubernetesFixture.createAndWait(namespace, null, deployment, service, true); + fabric8KubernetesFixture.createAndWait(namespace, deployment, service, true); } else { fabric8KubernetesFixture.deleteAndWait(namespace, configMap, null); diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java index 476aca831a..0583ef1179 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java @@ -17,14 +17,9 @@ package org.springframework.cloud.kubernetes.integration.tests.commons; import java.io.File; -import java.io.IOException; import java.io.InputStream; -import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.time.Duration; import java.util.Arrays; import java.util.List; @@ -33,7 +28,6 @@ import com.github.dockerjava.api.command.ListImagesCmd; import com.github.dockerjava.api.command.PullImageCmd; -import com.github.dockerjava.api.command.SaveImageCmd; import com.github.dockerjava.api.model.Image; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -50,9 +44,9 @@ import org.springframework.web.reactive.function.client.WebClient; import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.KUBERNETES_VERSION_FILE; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TEMP_FOLDER; import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TMP_IMAGES; import static org.springframework.cloud.kubernetes.integration.tests.commons.FixedPortsK3sContainer.CONTAINER; +import static org.springframework.cloud.kubernetes.integration.tests.commons.FixedPortsK3sContainer.REGISTRY_PORT; /** * A few commons things that can be re-used across clients. This is meant to be used for @@ -78,9 +72,11 @@ public static K3sContainer container() { return CONTAINER; } - public static void loadSpringCloudKubernetesImage(String project, K3sContainer container) { + public static void tagAndPushSpringCloudKubernetesImageToLocalDockerRegistry(String imageNameWithoutTag, + K3sContainer container) { try { - loadImage("springcloud/" + project, pomVersion(), project, container); + String springCloudImageWithTag = "springcloud/" + imageNameWithoutTag + ":" + pomVersion(); + tagAndPushImage(springCloudImageWithTag, container); } catch (Exception e) { throw new RuntimeException(e); @@ -88,56 +84,53 @@ public static void loadSpringCloudKubernetesImage(String project, K3sContainer c } /** - * create a tar, copy it in the running k3s and load this tar as an image. + * tag image and push to local registry. */ - public static void loadImage(String image, String tag, String tarName, K3sContainer container) { + public static void tagAndPushImage(String imageNameWithTag, K3sContainer container) { - if (imageAlreadyInK3s(container, tarName)) { + if (imageAlreadyInK3s(container, imageNameWithTag)) { return; } - // save image - try (SaveImageCmd saveImageCmd = container.getDockerClient().saveImageCmd(image)) { - InputStream imageStream = saveImageCmd.withTag(tag).exec(); - - Path imagePath = Paths.get(TEMP_FOLDER + "/" + tarName + ".tar"); - try { - Files.copy(imageStream, imagePath, StandardCopyOption.REPLACE_EXISTING); - } - catch (IOException e) { - throw new UncheckedIOException(e); + try { + int lastColon = imageNameWithTag.lastIndexOf(':'); + if (lastColon < 0) { + throw new IllegalArgumentException("image must include tag: " + imageNameWithTag); } - // import image with ctr. this works because TEMP_FOLDER is mounted in the - // container + + String imageWithoutTag = imageNameWithTag.substring(0, lastColon); + String tag = imageNameWithTag.substring(lastColon + 1); + + String targetRepository = "localhost:" + REGISTRY_PORT + "/" + imageWithoutTag; + String targetImageWithTag = targetRepository + ":" + tag; + + container.getDockerClient().tagImageCmd(imageNameWithTag, targetRepository, tag).exec(); + Awaitilities.awaitUntil(120, 1000, () -> { - Container.ExecResult result; try { - result = container.execInContainer("ctr", "i", "import", - Constants.TEMP_FOLDER + "/" + tarName + ".tar"); + container.getDockerClient().pushImageCmd(targetImageWithTag).start().awaitCompletion(); + return true; } catch (Exception e) { - throw new RuntimeException(e); + LOG.info("failed to push image " + targetImageWithTag + " to local registry", e); + return false; } - boolean noErrors = result.getStderr() == null || result.getStderr().isEmpty(); - if (!noErrors) { - LOG.info("error is : " + result.getStderr()); - } - return noErrors; }); } - + catch (Exception e) { + throw new RuntimeException(e); + } } /** - * Ensures a common external test image is available inside K3s/containerd. - * It first checks whether the image is already present in K3s. - * If not, it tries to load it as a tar under '/tmp/docker/images'. - * If no matching tar is found, it pulls the image directly inside K3s - * using 'ctr images pull'. - * This is meant for shared test images such as busybox, wiremock, kafka, etc. + * Ensures a common external test image is available inside K3s/containerd. It first + * checks whether the image is already present in K3s. If not, it tries to load it as + * a tar under '/tmp/docker/images'. If no matching tar is found, it pulls the image + * directly inside K3s using 'ctr images pull'. This is meant for shared test images + * such as busybox, wiremock, kafka, etc. */ - public static void loadOrPullCommonTestImages(K3sContainer container, String tarName, - String imageNameForDownload, String imageVersion) { + public static void loadOrPullCommonTestImages(K3sContainer container, String tarName, String imageNameForDownload, + String imageVersion) { if (imageAlreadyInK3s(container, tarName)) { return; @@ -225,7 +218,7 @@ private static String fullImageReference(String imageName, String imageVersion) } /** - * validates that the provided image does exist in the local docker registry. + * validates that the provided image does exist in the local dcoker cache. */ public static void validateImage(String image, K3sContainer container) { try (ListImagesCmd listImagesCmd = container.getDockerClient().listImagesCmd()) { @@ -239,15 +232,14 @@ public static void validateImage(String image, K3sContainer container) { } } - public static void pullImage(String image, String tag, String tarName, K3sContainer container) - throws InterruptedException { + public static void pullImage(String imageFromDeployment, K3sContainer container) throws InterruptedException { - if (imageAlreadyInK3s(container, tarName)) { + if (imageAlreadyInK3s(container, imageFromDeployment)) { return; } - try (PullImageCmd pullImageCmd = container.getDockerClient().pullImageCmd(image)) { - pullImageCmd.withTag(tag).start().awaitCompletion(); + try (PullImageCmd pullImageCmd = container.getDockerClient().pullImageCmd(imageFromDeployment)) { + pullImageCmd.start().awaitCompletion(); } } @@ -324,24 +316,21 @@ private static void loadImageFromPath(String tarName, K3sContainer container) { }); } - private static boolean imageAlreadyInK3s(K3sContainer container, String tarName) { + private static boolean imageAlreadyInK3s(K3sContainer container, String imageWithTag) { + try { + String stdout = container.execInContainer("ctr", "-n", "k8s.io", "images", "list", "-q").getStdout(); - if (tarName == null) { - return false; - } + boolean present = Arrays.stream(stdout.split("\\R")) + .map(String::trim) + .anyMatch(line -> line.contains(imageWithTag)); - try { - boolean present = container.execInContainer("sh", "-c", "ctr images list | grep " + tarName) - .getStdout() - .contains(tarName); if (present) { - System.out.println("image : " + tarName + " already in k3s, skipping"); + LOG.info("image : " + imageWithTag + " already in k3s, skipping"); return true; } - else { - System.out.println("image : " + tarName + " not in k3s"); - return false; - } + + LOG.info("image : " + imageWithTag + " not in k3s"); + return false; } catch (Exception e) { throw new RuntimeException(e); diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Constants.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Constants.java index 811299300b..c3c895667f 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Constants.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Constants.java @@ -16,8 +16,6 @@ package org.springframework.cloud.kubernetes.integration.tests.commons; -import java.io.File; - /** * @author wind57 */ @@ -32,11 +30,6 @@ private Constants() { */ static final String TMP_IMAGES = "/tmp/docker/images"; - /** - * Temporary folder where to load images. - */ - static final String TEMP_FOLDER = new File(System.getProperty("java.io.tmpdir")).getAbsolutePath(); - /** * where is the version situated. */ diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedIdNetworkProvider.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedIdNetworkProvider.java new file mode 100644 index 0000000000..8c40d131cc --- /dev/null +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedIdNetworkProvider.java @@ -0,0 +1,58 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.integration.tests.commons; + +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.Network; + +/** + * re-use implementation for a Network. It first checks if such a named network already + * exists via docker client API and creates a new one if it does not. + * + * @author wind57 + */ +final class FixedIdNetworkProvider { + + private FixedIdNetworkProvider() { + + } + + static Network createReusableNetwork(String name) { + var client = DockerClientFactory.instance().client(); + + String id = client.listNetworksCmd() + .exec() + .stream() + .filter(network -> name.equals(network.getName())) + .map(com.github.dockerjava.api.model.Network::getId) + .findFirst() + .orElseGet(() -> client.createNetworkCmd().withName(name).withCheckDuplicate(true).exec().getId()); + + return new org.testcontainers.containers.Network() { + @Override + public String getId() { + return id; + } + + @Override + public void close() { + // intentionally no-op + } + }; + } + +} diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java index d935f3c28e..e6104d6e5d 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java @@ -20,11 +20,14 @@ import com.github.dockerjava.api.model.Bind; import com.github.dockerjava.api.model.HostConfig; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; import org.testcontainers.k3s.K3sContainer; import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.MountableFile; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TEMP_FOLDER; import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TMP_IMAGES; +import static org.springframework.cloud.kubernetes.integration.tests.commons.FixedIdNetworkProvider.createReusableNetwork; /** * A K3sContainer, but with fixed port mappings. This is needed because of the nature of @@ -32,49 +35,99 @@ * * @author wind57 */ -final class FixedPortsK3sContainer extends K3sContainer { +public final class FixedPortsK3sContainer extends K3sContainer { /** * Test containers exposed ports. */ private static final int[] EXPOSED_PORTS = new int[] { 80, 6443, 8080, 8888, 9092, 32321, 32322 }; + /** + * Port for the local running images registry. + */ + public static final int REGISTRY_PORT = 6000; + + /** + * Small doc on how the set-up works. ( 5000 is just an example )
+ * - we start a local registry and expose it on localhost:<5000>
+ * - from the host, we can push an image with:
+ * docker push localhost:5000/image:tag
+ * - the image is now stored in that local registry
+ * - k3s later sees the image reference: localhost:5000/image:tag
+ * - inside the k3s container, localhost would point to k3s itself, not to the registry
+ * - because of the mirror entry in registries.yaml, when k3s/containerd sees
+ * an image starting with localhost:5000, it actually uses the endpoint 'http://registry:5000'
+ * - LocalRegistryContainer has withNetworkAliases("registry")
+ * - this makes the registry reachable from the same Docker network via the hostname "registry"
+ *
+ */
+ private static final Network NETWORK = createReusableNetwork("spring-cloud-kubernetes-local-docker-registry");
+
+ private static final LocalRegistryContainer REGISTRY = new LocalRegistryContainer().configureFixedPorts()
+ .withNetwork(NETWORK)
+ .withNetworkAliases("registry")
+ .withReuse(true);
+
/**
* Rancher version to use for test-containers.
*/
private static final String RANCHER_VERSION = "rancher/k3s:v1.35.4-k3s1";
+ /**
+ * Docker registry version.
+ */
+ private static final String DOCKER_REGISTRY_VERSION = "registry:3.1.0";
+
/**
* Command to use when starting rancher. Without "server" option, traefik is not
* installed
*/
- private static final String RANCHER_COMMAND = "server --disable=metric-server";
+ private static final String RANCHER_COMMAND = "server --disable=metric-server --disable-default-registry-endpoint";
static final K3sContainer CONTAINER = new FixedPortsK3sContainer(DockerImageName.parse(RANCHER_VERSION))
.configureFixedPorts()
.addBinds()
.withCommand(RANCHER_COMMAND)
- .withReuse(true);
+ .withReuse(true)
+ .withNetwork(NETWORK)
+ .withCopyFileToContainer(MountableFile.forClasspathResource("registries.yaml"),
+ "/etc/rancher/k3s/registries.yaml");
- FixedPortsK3sContainer(DockerImageName dockerImageName) {
+ private FixedPortsK3sContainer(DockerImageName dockerImageName) {
super(dockerImageName);
+ REGISTRY.start();
}
- FixedPortsK3sContainer configureFixedPorts() {
+ private FixedPortsK3sContainer configureFixedPorts() {
for (int port : EXPOSED_PORTS) {
super.addFixedExposedPort(port, port);
}
return this;
}
- FixedPortsK3sContainer addBinds() {
+ private FixedPortsK3sContainer addBinds() {
super.withCreateContainerCmdModifier(cmd -> {
HostConfig hostConfig = Objects.requireNonNull(cmd.getHostConfig());
- hostConfig.withBinds(Bind.parse(TEMP_FOLDER + ":" + TEMP_FOLDER),
- Bind.parse(TMP_IMAGES + ":" + TMP_IMAGES));
+ hostConfig.withBinds(Bind.parse(TMP_IMAGES + ":" + TMP_IMAGES));
});
return this;
}
+ /**
+ * official registry image with fixed port 5000.
+ */
+ private static final class LocalRegistryContainer extends GenericContainer