first test run

Signed-off-by: wind57 <eugen.rabii@gmail.com>

Author: wind57

spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/reload/TestAssertions.java

@@ -96,7 +96,7 @@ static void manifests(Phase phase, Fabric8ClientKubernetesFixture fabric8Kuberne
 
 		if (phase.equals(Phase.CREATE)) {
 			fabric8KubernetesFixture.createAndWait(namespace, configMap, null);
-			fabric8KubernetesFixture.createAndWait(namespace, null, deployment, service, true);
+			fabric8KubernetesFixture.createAndWait(namespace, deployment, service, true);
 		}
 		else {
 			fabric8KubernetesFixture.deleteAndWait(namespace, configMap, null);

spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java

@@ -17,14 +17,9 @@
 package org.springframework.cloud.kubernetes.integration.tests.commons;
 
 import java.io.File;
-import java.io.IOException;
 import java.io.InputStream;
-import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
 import java.time.Duration;
 import java.util.Arrays;
 import java.util.List;
@@ -33,7 +28,6 @@
 
 import com.github.dockerjava.api.command.ListImagesCmd;
 import com.github.dockerjava.api.command.PullImageCmd;
-import com.github.dockerjava.api.command.SaveImageCmd;
 import com.github.dockerjava.api.model.Image;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -50,9 +44,9 @@
 import org.springframework.web.reactive.function.client.WebClient;
 
 import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.KUBERNETES_VERSION_FILE;
-import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TEMP_FOLDER;
 import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TMP_IMAGES;
 import static org.springframework.cloud.kubernetes.integration.tests.commons.FixedPortsK3sContainer.CONTAINER;
+import static org.springframework.cloud.kubernetes.integration.tests.commons.FixedPortsK3sContainer.REGISTRY_PORT;
 
 /**
  * A few commons things that can be re-used across clients. This is meant to be used for
@@ -78,66 +72,64 @@ public static K3sContainer container() {
 		return CONTAINER;
 	}
 
-	public static void loadSpringCloudKubernetesImage(String project, K3sContainer container) {
+	public static void tagAndPushSpringCloudKubernetesImage(String imageName, K3sContainer container) {
 		try {
-			loadImage("springcloud/" + project, pomVersion(), project, container);
+			String springCloudImage = "springcloud/" + imageName + ":" + pomVersion();
+			tagAndPushImage(springCloudImage, container);
 		}
 		catch (Exception e) {
 			throw new RuntimeException(e);
 		}
 	}
 
 	/**
-	 * create a tar, copy it in the running k3s and load this tar as an image.
+	 * tag image and push to local registry.
 	 */
-	public static void loadImage(String image, String tag, String tarName, K3sContainer container) {
+	public static void tagAndPushImage(String imageFromDeploymentWithTag, K3sContainer container) {
 
-		if (imageAlreadyInK3s(container, tarName)) {
+		if (imageAlreadyInK3s(container, imageFromDeploymentWithTag)) {
 			return;
 		}
 
-		// save image
-		try (SaveImageCmd saveImageCmd = container.getDockerClient().saveImageCmd(image)) {
-			InputStream imageStream = saveImageCmd.withTag(tag).exec();
-
-			Path imagePath = Paths.get(TEMP_FOLDER + "/" + tarName + ".tar");
-			try {
-				Files.copy(imageStream, imagePath, StandardCopyOption.REPLACE_EXISTING);
-			}
-			catch (IOException e) {
-				throw new UncheckedIOException(e);
+		try {
+			int lastColon = imageFromDeploymentWithTag.lastIndexOf(':');
+			if (lastColon < 0) {
+				throw new IllegalArgumentException("image must include tag: " + imageFromDeploymentWithTag);
 			}
-			// import image with ctr. this works because TEMP_FOLDER is mounted in the
-			// container
+
+			String imageWithoutTag = imageFromDeploymentWithTag.substring(0, lastColon);
+			String tag = imageFromDeploymentWithTag.substring(lastColon + 1);
+
+			String targetRepository = "localhost:" + REGISTRY_PORT + "/" + imageWithoutTag;
+			String targetImageWithTag = targetRepository + ":" + tag;
+
+			container.getDockerClient().tagImageCmd(imageFromDeploymentWithTag, targetRepository, tag).exec();
+
 			Awaitilities.awaitUntil(120, 1000, () -> {
-				Container.ExecResult result;
 				try {
-					result = container.execInContainer("ctr", "i", "import",
-							Constants.TEMP_FOLDER + "/" + tarName + ".tar");
+					container.getDockerClient().pushImageCmd(targetImageWithTag).start().awaitCompletion();
+					return true;
 				}
 				catch (Exception e) {
-					throw new RuntimeException(e);
+					LOG.info("failed to push image " + targetImageWithTag + " to local registry", e);
+					return false;
 				}
-				boolean noErrors = result.getStderr() == null || result.getStderr().isEmpty();
-				if (!noErrors) {
-					LOG.info("error is : " + result.getStderr());
-				}
-				return noErrors;
 			});
 		}
-
+		catch (Exception e) {
+			throw new RuntimeException(e);
+		}
 	}
 
 	/**
-	 * Ensures a common external test image is available inside K3s/containerd.
-	 * It first checks whether the image is already present in K3s.
-	 * If not, it tries to load it as a tar under '/tmp/docker/images'.
-	 * If no matching tar is found, it pulls the image directly inside K3s
-	 * using 'ctr images pull'.
-	 * This is meant for shared test images such as busybox, wiremock, kafka, etc.
+	 * Ensures a common external test image is available inside K3s/containerd. It first
+	 * checks whether the image is already present in K3s. If not, it tries to load it as
+	 * a tar under '/tmp/docker/images'. If no matching tar is found, it pulls the image
+	 * directly inside K3s using 'ctr images pull'. This is meant for shared test images
+	 * such as busybox, wiremock, kafka, etc.
 	 */
-	public static void loadOrPullCommonTestImages(K3sContainer container, String tarName,
-			String imageNameForDownload, String imageVersion) {
+	public static void loadOrPullCommonTestImages(K3sContainer container, String tarName, String imageNameForDownload,
+			String imageVersion) {
 
 		if (imageAlreadyInK3s(container, tarName)) {
 			return;
@@ -239,15 +231,14 @@ public static void validateImage(String image, K3sContainer container) {
 		}
 	}
 
-	public static void pullImage(String image, String tag, String tarName, K3sContainer container)
-			throws InterruptedException {
+	public static void pullImage(String imageFromDeployment, K3sContainer container) throws InterruptedException {
 
-		if (imageAlreadyInK3s(container, tarName)) {
+		if (imageAlreadyInK3s(container, imageFromDeployment)) {
 			return;
 		}
 
-		try (PullImageCmd pullImageCmd = container.getDockerClient().pullImageCmd(image)) {
-			pullImageCmd.withTag(tag).start().awaitCompletion();
+		try (PullImageCmd pullImageCmd = container.getDockerClient().pullImageCmd(imageFromDeployment)) {
+			pullImageCmd.start().awaitCompletion();
 		}
 	}
 
@@ -324,24 +315,19 @@ private static void loadImageFromPath(String tarName, K3sContainer container) {
 		});
 	}
 
-	private static boolean imageAlreadyInK3s(K3sContainer container, String tarName) {
+	private static boolean imageAlreadyInK3s(K3sContainer container, String imageWithTag) {
+		try {
+			String stdout = container.execInContainer("ctr", "-n", "k8s.io", "images", "list", "-q").getStdout();
 
-		if (tarName == null) {
-			return false;
-		}
+			boolean present = Arrays.stream(stdout.split("\\R")).map(String::trim).anyMatch(imageWithTag::equals);
 
-		try {
-			boolean present = container.execInContainer("sh", "-c", "ctr images list | grep " + tarName)
-				.getStdout()
-				.contains(tarName);
 			if (present) {
-				System.out.println("image : " + tarName + " already in k3s, skipping");
+				System.out.println("image : " + imageWithTag + " already in k3s, skipping");
 				return true;
 			}
-			else {
-				System.out.println("image : " + tarName + " not in k3s");
-				return false;
-			}
+
+			System.out.println("image : " + imageWithTag + " not in k3s");
+			return false;
 		}
 		catch (Exception e) {
 			throw new RuntimeException(e);

spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Constants.java

@@ -16,8 +16,6 @@
 
 package org.springframework.cloud.kubernetes.integration.tests.commons;
 
-import java.io.File;
-
 /**
  * @author wind57
  */
@@ -32,11 +30,6 @@ private Constants() {
 	 */
 	static final String TMP_IMAGES = "/tmp/docker/images";
 
-	/**
-	 * Temporary folder where to load images.
-	 */
-	static final String TEMP_FOLDER = new File(System.getProperty("java.io.tmpdir")).getAbsolutePath();
-
 	/**
 	 * where is the version situated.
 	 */

spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java

@@ -16,14 +16,22 @@
 
 package org.springframework.cloud.kubernetes.integration.tests.commons;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Objects;
 
 import com.github.dockerjava.api.model.Bind;
 import com.github.dockerjava.api.model.HostConfig;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
 import org.testcontainers.k3s.K3sContainer;
 import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.MountableFile;
+
+import org.springframework.test.util.TestSocketUtils;
 
-import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TEMP_FOLDER;
 import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TMP_IMAGES;
 
 /**
@@ -32,49 +40,114 @@
  *
  * @author wind57
  */
-final class FixedPortsK3sContainer extends K3sContainer {
+public final class FixedPortsK3sContainer extends K3sContainer {
 
 	/**
 	 * Test containers exposed ports.
 	 */
 	private static final int[] EXPOSED_PORTS = new int[] { 80, 6443, 8080, 8888, 9092, 32321, 32322 };
 
+	/**
+	 * Port for the local running images registry.
+	 */
+	public static final int REGISTRY_PORT = TestSocketUtils.findAvailableTcpPort();
+
+	private static final Network NETWORK = Network.newNetwork();
+
+	private static final Path REGISTRIES_YAML = createRegistriesYaml();
+
+	private static final LocalRegistryContainer REGISTRY = new LocalRegistryContainer().configureFixedPorts()
+		.withNetwork(NETWORK)
+		.withNetworkAliases("registry")
+		.withReuse(true);
+
 	/**
 	 * Rancher version to use for test-containers.
 	 */
 	private static final String RANCHER_VERSION = "rancher/k3s:v1.34.1-k3s1";
 
+	/**
+	 * Docker registry version.
+	 */
+	private static final String DOCKER_REGISTRY_VERSION = "registry:3.1.0";
+
 	/**
 	 * Command to use when starting rancher. Without "server" option, traefik is not
 	 * installed
 	 */
-	private static final String RANCHER_COMMAND = "server --disable=metric-server";
+	private static final String RANCHER_COMMAND = "server --disable=metric-server --disable-default-registry-endpoint";
 
 	static final K3sContainer CONTAINER = new FixedPortsK3sContainer(DockerImageName.parse(RANCHER_VERSION))
 		.configureFixedPorts()
 		.addBinds()
 		.withCommand(RANCHER_COMMAND)
-		.withReuse(true);
+		.withReuse(true)
+		.withNetwork(NETWORK)
+		.withCopyFileToContainer(MountableFile.forHostPath(REGISTRIES_YAML), "/etc/rancher/k3s/registries.yaml");
 
-	FixedPortsK3sContainer(DockerImageName dockerImageName) {
+	private FixedPortsK3sContainer(DockerImageName dockerImageName) {
 		super(dockerImageName);
+		REGISTRY.start();
 	}
 
-	FixedPortsK3sContainer configureFixedPorts() {
+	private FixedPortsK3sContainer configureFixedPorts() {
 		for (int port : EXPOSED_PORTS) {
 			super.addFixedExposedPort(port, port);
 		}
 		return this;
 	}
 
-	FixedPortsK3sContainer addBinds() {
+	private FixedPortsK3sContainer addBinds() {
 		super.withCreateContainerCmdModifier(cmd -> {
 			HostConfig hostConfig = Objects.requireNonNull(cmd.getHostConfig());
-			hostConfig.withBinds(Bind.parse(TEMP_FOLDER + ":" + TEMP_FOLDER),
-					Bind.parse(TMP_IMAGES + ":" + TMP_IMAGES));
+			hostConfig.withBinds(Bind.parse(TMP_IMAGES + ":" + TMP_IMAGES));
 		});
 
 		return this;
 	}
 
+	/**
+	 * Small doc on how the set-up works. ( 5000 is just an example ) <pre>
+	 *     - we start a local registry and expose it on localhost:<5000>
+	 *     - from the host, we can push an image with:
+	 *       docker push localhost:5000/image:tag
+	 *     - the image is now stored in that local registry
+	 *     - k3s later sees the image reference: localhost:5000/image:tag
+	 *     - inside the k3s container, localhost would point to k3s itself, not to the registry
+	 *     - because of the mirror entry in registries.yaml, when k3s/containerd sees
+	 *       an image starting with localhost:5000, it actually uses the endpoint 'http://registry:5000'
+	 *     - LocalRegistryContainer has withNetworkAliases("registry")
+	 *     - this makes the registry reachable from the same Docker network via the hostname "registry"
+	 * </pre>
+	 */
+	private static Path createRegistriesYaml() {
+		try {
+			Path file = Files.createTempFile("registries", ".yaml");
+			// can't use text blocks here because of checkstyle
+			String content = "mirrors:\n" + "  \"localhost:%s\":\n" + "    endpoint:\n"
+					+ "      - \"http://registry:5000\"\n";
+			Files.writeString(file, content.formatted(REGISTRY_PORT));
+			return file;
+		}
+		catch (IOException e) {
+			throw new UncheckedIOException(e);
+		}
+	}
+
+	/**
+	 * official registry image with fixed port 5000.
+	 */
+	private static final class LocalRegistryContainer extends GenericContainer<LocalRegistryContainer> {
+
+		private LocalRegistryContainer() {
+			super(DOCKER_REGISTRY_VERSION);
+		}
+
+		private LocalRegistryContainer configureFixedPorts() {
+			addFixedExposedPort(REGISTRY_PORT, 5000);
+			return this;
+		}
+
+	}
+
 }