diff --git a/.github/workflows/composites/pre-test-actions/action.yaml b/.github/workflows/composites/pre-test-actions/action.yaml index 962c29c79a..51b4866e49 100644 --- a/.github/workflows/composites/pre-test-actions/action.yaml +++ b/.github/workflows/composites/pre-test-actions/action.yaml @@ -29,7 +29,9 @@ runs: with: jdk-version: ${{ inputs.jdk-version }} - - name: restore common images + # restore or download common docker test images, like: busybox, wiremock. + # these will be placed in /tmp/docker/images + - name: restore common images to /tmp/docker/images uses: ./.github/workflows/composites/restore-docker-images - name: build project @@ -37,18 +39,47 @@ runs: run: | ./mvnw clean install -Dspring-boot.build-image.skip=true -DskipITs -DskipTests -T1C -U -B -q + # we could cache these images also, but the vast majority of time, we will + # be running against a SNAPSHOT version. - name: build controllers project uses: ./.github/workflows/composites/build-controllers-project + # we could cache these images also, but the vast majority of time, we will + # be running against a SNAPSHOT version. - name: build integration tests project uses: ./.github/workflows/composites/build-integration-tests-project + - name: put controllers and integration tests tars to /tmp/docker/images + shell: bash + run: | + + # find the version of our project from file: spring-cloud-kubernetes-integration-tests/pom.xml + PROJECT_VERSION="$(./mvnw -q -f spring-cloud-kubernetes-integration-tests/pom.xml help:evaluate -Dexpression=project.version -DforceStdout)" + + # get the existing docker images ( controllers and integration tests ) into the 'images' array + mapfile -t images < <( + docker image ls --format '{{.Repository}}:{{.Tag}}' \ + | grep "^springcloud/.*:${PROJECT_VERSION}$" \ + | sort -u + ) + + # print what we currently have + for i in "${!images[@]}"; do + printf 'images[%s]=%s\n' "$i" "${images[$i]}" + done + + # look at variable 'var', if it starts with 'springcloud/', remove it, return the rest. + for image in "${images[@]}"; do + tar_name="${image#springcloud/}.tar" + docker save -o "/tmp/docker/images/${tar_name}" "$image" + done + + - name: display /tmp/docker/images + shell: bash + run: ls -l /tmp/docker/images + - name: download tests uses: actions/download-artifact@v4 with: name: tests-${{ inputs.branch }}-jdk${{ inputs.jdk-version }}.txt path: /tmp - - - - diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/SecretWatcherWithLabelsTest.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/SecretWatcherWithLabelsTest.java index 95c1e9d6b5..43eb55bbfd 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/SecretWatcherWithLabelsTest.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/SecretWatcherWithLabelsTest.java @@ -67,7 +67,6 @@ "spring.cloud.kubernetes.reload.monitoring-config-maps=false", "spring.cloud.kubernetes.reload.monitoring-secrets=true", "spring.cloud.kubernetes.secrets.enabled=true", - "spring.cloud.kubernetes.secrets.enabled=true", "spring.cloud.kubernetes.reload.secrets-labels[spring.cloud.kubernetes.secret]=true", "spring.cloud.kubernetes.configuration.watcher.refresh-delay=1ms" }) class SecretWatcherWithLabelsTest { diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/reload/TestAssertions.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/reload/TestAssertions.java index 2280049998..5ec746884c 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/reload/TestAssertions.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/reload/TestAssertions.java @@ -96,7 +96,7 @@ static void manifests(Phase phase, Fabric8ClientKubernetesFixture fabric8Kuberne if (phase.equals(Phase.CREATE)) { fabric8KubernetesFixture.createAndWait(namespace, configMap, null); - fabric8KubernetesFixture.createAndWait(namespace, null, deployment, service, true); + fabric8KubernetesFixture.createAndWait(namespace, deployment, service, true); } else { fabric8KubernetesFixture.deleteAndWait(namespace, configMap, null); diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-kafka-configmap-reload/kafka-configmap-test-app/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/multiple/apps/ConfigurationWatcherBusKafkaIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-kafka-configmap-reload/kafka-configmap-test-app/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/multiple/apps/ConfigurationWatcherBusKafkaIT.java index b39b3973e9..95e5ec883c 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-kafka-configmap-reload/kafka-configmap-test-app/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/multiple/apps/ConfigurationWatcherBusKafkaIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-kafka-configmap-reload/kafka-configmap-test-app/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/multiple/apps/ConfigurationWatcherBusKafkaIT.java @@ -53,7 +53,7 @@ class ConfigurationWatcherBusKafkaIT { void setup(NativeClientKubernetesFixture fixture) { V1Deployment deployment = fixture.yaml("app/app-deployment.yaml", V1Deployment.class); V1Service service = fixture.yaml("app/app-service.yaml", V1Service.class); - fixture.createAndWait("default", null, deployment, service, true); + fixture.createAndWait("default", deployment, service, true); } @AfterEach diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-rabbitmq-secret-reload/rabbitmq-secret-test-app/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/multiple/apps/ConfigurationWatcherBusAmqpIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-rabbitmq-secret-reload/rabbitmq-secret-test-app/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/multiple/apps/ConfigurationWatcherBusAmqpIT.java index e38e551deb..400fceeddd 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-rabbitmq-secret-reload/rabbitmq-secret-test-app/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/multiple/apps/ConfigurationWatcherBusAmqpIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-rabbitmq-secret-reload/rabbitmq-secret-test-app/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/multiple/apps/ConfigurationWatcherBusAmqpIT.java @@ -51,7 +51,7 @@ class ConfigurationWatcherBusAmqpIT { void setup(NativeClientKubernetesFixture fixture) { V1Deployment deployment = fixture.yaml("app/app-deployment.yaml", V1Deployment.class); V1Service service = fixture.yaml("app/app-service.yaml", V1Service.class); - fixture.createAndWait("default", null, deployment, service, true); + fixture.createAndWait("default", deployment, service, true); } @AfterEach diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientConfigMapConfigTreeIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientConfigMapConfigTreeIT.java index e13f9e5bd3..fab94e7bfe 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientConfigMapConfigTreeIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientConfigMapConfigTreeIT.java @@ -50,12 +50,12 @@ class K8sClientConfigMapConfigTreeIT extends K8sClientReloadBase { @BeforeAll static void beforeAllLocal(NativeClientKubernetesFixture fixture) { - manifests(Phase.CREATE, fixture, "default", "spring-cloud-kubernetes-k8s-client-reload"); + manifests(Phase.CREATE, fixture, "default"); } @AfterAll static void afterAll(NativeClientKubernetesFixture fixture) { - manifests(Phase.DELETE, fixture, "default", "spring-cloud-kubernetes-k8s-client-reload"); + manifests(Phase.DELETE, fixture, "default"); } /** diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientReloadBase.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientReloadBase.java index 0267ebc86b..dc593166f6 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientReloadBase.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientReloadBase.java @@ -88,8 +88,7 @@ protected static void replaceConfigMap(CoreV1Api api, V1ConfigMap configMap) { } } - protected static void manifests(Phase phase, NativeClientKubernetesFixture fixture, String namespace, - String imageName) { + protected static void manifests(Phase phase, NativeClientKubernetesFixture fixture, String namespace) { V1Deployment deployment = fixture.yaml("mount/deployment.yaml", V1Deployment.class); V1Service service = fixture.yaml("mount/service.yaml", V1Service.class); @@ -97,7 +96,7 @@ protected static void manifests(Phase phase, NativeClientKubernetesFixture fixtu if (phase.equals(Phase.CREATE)) { fixture.createAndWait(namespace, configMap, null); - fixture.createAndWait(namespace, imageName, deployment, service, true); + fixture.createAndWait(namespace, deployment, service, true); } else { fixture.deleteAndWait(namespace, configMap, null); @@ -114,7 +113,7 @@ protected static void manifestsSecret(Phase phase, NativeClientKubernetesFixture if (phase.equals(Phase.CREATE)) { fixture.createAndWait("default", null, secret); - fixture.createAndWait("default", "spring-cloud-kubernetes-k8s-client-reload", deployment, service, true); + fixture.createAndWait("default", deployment, service, true); } else { fixture.deleteAndWait("default", null, secret); diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java index 476aca831a..1b3ff46beb 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java @@ -16,28 +16,11 @@ package org.springframework.cloud.kubernetes.integration.tests.commons; -import java.io.File; -import java.io.IOException; import java.io.InputStream; -import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.time.Duration; -import java.util.Arrays; -import java.util.List; import java.util.Objects; -import java.util.Optional; -import com.github.dockerjava.api.command.ListImagesCmd; -import com.github.dockerjava.api.command.PullImageCmd; -import com.github.dockerjava.api.command.SaveImageCmd; -import com.github.dockerjava.api.model.Image; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.testcontainers.containers.Container; import org.testcontainers.k3s.K3sContainer; import reactor.netty.http.client.HttpClient; import reactor.util.retry.Retry; @@ -50,8 +33,6 @@ import org.springframework.web.reactive.function.client.WebClient; import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.KUBERNETES_VERSION_FILE; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TEMP_FOLDER; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TMP_IMAGES; import static org.springframework.cloud.kubernetes.integration.tests.commons.FixedPortsK3sContainer.CONTAINER; /** @@ -64,12 +45,6 @@ public final class Commons { private static String POM_VERSION; - private static final Log LOG = LogFactory.getLog(Commons.class); - - private static final String DOCKER_IO = "docker.io/"; - - private static final String DOCKER_IO_LIBRARY = DOCKER_IO + "library/"; - private Commons() { throw new AssertionError("No instance provided"); } @@ -78,179 +53,6 @@ public static K3sContainer container() { return CONTAINER; } - public static void loadSpringCloudKubernetesImage(String project, K3sContainer container) { - try { - loadImage("springcloud/" + project, pomVersion(), project, container); - } - catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * create a tar, copy it in the running k3s and load this tar as an image. - */ - public static void loadImage(String image, String tag, String tarName, K3sContainer container) { - - if (imageAlreadyInK3s(container, tarName)) { - return; - } - - // save image - try (SaveImageCmd saveImageCmd = container.getDockerClient().saveImageCmd(image)) { - InputStream imageStream = saveImageCmd.withTag(tag).exec(); - - Path imagePath = Paths.get(TEMP_FOLDER + "/" + tarName + ".tar"); - try { - Files.copy(imageStream, imagePath, StandardCopyOption.REPLACE_EXISTING); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - // import image with ctr. this works because TEMP_FOLDER is mounted in the - // container - Awaitilities.awaitUntil(120, 1000, () -> { - Container.ExecResult result; - try { - result = container.execInContainer("ctr", "i", "import", - Constants.TEMP_FOLDER + "/" + tarName + ".tar"); - } - catch (Exception e) { - throw new RuntimeException(e); - } - boolean noErrors = result.getStderr() == null || result.getStderr().isEmpty(); - if (!noErrors) { - LOG.info("error is : " + result.getStderr()); - } - return noErrors; - }); - } - - } - - /** - * Ensures a common external test image is available inside K3s/containerd. - * It first checks whether the image is already present in K3s. - * If not, it tries to load it as a tar under '/tmp/docker/images'. - * If no matching tar is found, it pulls the image directly inside K3s - * using 'ctr images pull'. - * This is meant for shared test images such as busybox, wiremock, kafka, etc. - */ - public static void loadOrPullCommonTestImages(K3sContainer container, String tarName, - String imageNameForDownload, String imageVersion) { - - if (imageAlreadyInK3s(container, tarName)) { - return; - } - - File dockerImagesRootDir = Paths.get(TMP_IMAGES).toFile(); - if (dockerImagesRootDir.exists() && dockerImagesRootDir.isDirectory()) { - File[] tars = dockerImagesRootDir.listFiles(); - if (tars != null && tars.length > 0) { - Optional found = Arrays.stream(tars) - .map(File::getName) - .filter(x -> x.contains(tarName)) - .findFirst(); - if (found.isPresent()) { - LOG.info("running in github actions, will load from : " + TMP_IMAGES + " tar : " + found.get()); - loadImageFromPath(found.get(), container); - return; - } - else { - LOG.info(tarName + " not found, resorting to pulling the image"); - } - } - else { - LOG.info("no tars found, will resort to pulling the image"); - } - } - else { - LOG.info("running outside github actions"); - } - - try { - LOG.info("pulling image inside k3s to avoid Docker save/ctr import compatibility issues"); - LOG.info("using : " + imageVersion + " for : " + imageNameForDownload); - pullImageInsideK3s(container, imageNameForDownload, imageVersion); - } - catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * Pull image directly inside the K3s container via ctr (containerd). This avoids - * Docker save + ctr import, which can fail with "content digest not found" for - * multi-platform or OCI images. If DOCKER_HUB_USERNAME and DOCKER_HUB_PASSWORD are - * set, they are passed as {@code --user username:password} for registry auth. - */ - private static void pullImageInsideK3s(K3sContainer container, String imageNameForDownload, String imageVersion) { - String fullImageRef = fullImageReference(imageNameForDownload, imageVersion); - - final String[] ctrArgs = buildCtrPullArgs(fullImageRef); - Awaitilities.awaitUntil(120, 1, () -> { - try { - Container.ExecResult result = container.execInContainer(ctrArgs); - boolean noErrors = result.getStderr() == null || result.getStderr().isEmpty(); - if (!noErrors) { - LOG.info("ctr pull stderr: " + result.getStderr()); - } - return noErrors; - } - catch (Exception e) { - throw new RuntimeException(e); - } - - }); - } - - private static String[] buildCtrPullArgs(String fullImageRef) { - String username = System.getenv("DOCKER_HUB_USERNAME"); - String password = System.getenv("DOCKER_HUB_PASSWORD"); - if (username != null && !username.isBlank() && password != null && !password.isBlank()) { - LOG.info("pulling inside k3s with Docker Hub credentials: " + fullImageRef); - return new String[] { "ctr", "-n", "k8s.io", "images", "pull", "--user", username + ":" + password, - fullImageRef }; - } - LOG.info("pulling inside k3s: " + fullImageRef); - return new String[] { "ctr", "-n", "k8s.io", "images", "pull", fullImageRef }; - } - - private static String fullImageReference(String imageName, String imageVersion) { - String imageNameAndVersion = imageName + ":" + imageVersion; - if (imageName.contains("/")) { - return DOCKER_IO + imageNameAndVersion; - } - return DOCKER_IO_LIBRARY + imageNameAndVersion; - } - - /** - * validates that the provided image does exist in the local docker registry. - */ - public static void validateImage(String image, K3sContainer container) { - try (ListImagesCmd listImagesCmd = container.getDockerClient().listImagesCmd()) { - List images = listImagesCmd.exec(); - images.stream() - .filter(x -> Arrays.stream(x.getRepoTags() == null ? new String[] {} : x.getRepoTags()) - .anyMatch(y -> y.contains(image))) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Image : " + image + " not build locally. " - + "You need to build it first, and then run the test")); - } - } - - public static void pullImage(String image, String tag, String tarName, K3sContainer container) - throws InterruptedException { - - if (imageAlreadyInK3s(container, tarName)) { - return; - } - - try (PullImageCmd pullImageCmd = container.getDockerClient().pullImageCmd(image)) { - pullImageCmd.withTag(tag).start().awaitCompletion(); - } - } - public static String pomVersion() { if (POM_VERSION == null) { try (InputStream in = new ClassPathResource(KUBERNETES_VERSION_FILE).getInputStream()) { @@ -307,45 +109,4 @@ public static RetryBackoffSpec retrySpec() { return Retry.fixedDelay(15, Duration.ofSeconds(1)).filter(Objects::nonNull); } - private static void loadImageFromPath(String tarName, K3sContainer container) { - Awaitilities.awaitUntil(120, 1000, () -> { - Container.ExecResult result; - try { - result = container.execInContainer("ctr", "i", "import", Constants.TMP_IMAGES + "/" + tarName); - } - catch (Exception e) { - throw new RuntimeException(e); - } - boolean noErrors = result.getStderr() == null || result.getStderr().isEmpty(); - if (!noErrors) { - LOG.info("error is : " + result.getStderr()); - } - return noErrors; - }); - } - - private static boolean imageAlreadyInK3s(K3sContainer container, String tarName) { - - if (tarName == null) { - return false; - } - - try { - boolean present = container.execInContainer("sh", "-c", "ctr images list | grep " + tarName) - .getStdout() - .contains(tarName); - if (present) { - System.out.println("image : " + tarName + " already in k3s, skipping"); - return true; - } - else { - System.out.println("image : " + tarName + " not in k3s"); - return false; - } - } - catch (Exception e) { - throw new RuntimeException(e); - } - } - } diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Constants.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Constants.java index 811299300b..80b2d87250 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Constants.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Constants.java @@ -28,14 +28,21 @@ private Constants() { } /** - * this path is generated by the pipeline of github actions. + * Directory populated by the CI pipeline with prebuilt image tar files. It contains + * tar files for: + * */ - static final String TMP_IMAGES = "/tmp/docker/images"; + static final String CI_IMAGE_TARS_DIR = "/tmp/docker/images"; /** - * Temporary folder where to load images. + * Directory used during local runs to stage image tar files created from the local + * Docker cache before importing them into K3s with 'ctr i import'. */ - static final String TEMP_FOLDER = new File(System.getProperty("java.io.tmpdir")).getAbsolutePath(); + static final String LOCAL_IMAGE_TARS_DIR = new File(System.getProperty("java.io.tmpdir")).getAbsolutePath(); /** * where is the version situated. diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java index d935f3c28e..283ae9a609 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java @@ -23,8 +23,8 @@ import org.testcontainers.k3s.K3sContainer; import org.testcontainers.utility.DockerImageName; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TEMP_FOLDER; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.TMP_IMAGES; +import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.CI_IMAGE_TARS_DIR; +import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.LOCAL_IMAGE_TARS_DIR; /** * A K3sContainer, but with fixed port mappings. This is needed because of the nature of @@ -70,8 +70,8 @@ FixedPortsK3sContainer configureFixedPorts() { FixedPortsK3sContainer addBinds() { super.withCreateContainerCmdModifier(cmd -> { HostConfig hostConfig = Objects.requireNonNull(cmd.getHostConfig()); - hostConfig.withBinds(Bind.parse(TEMP_FOLDER + ":" + TEMP_FOLDER), - Bind.parse(TMP_IMAGES + ":" + TMP_IMAGES)); + hostConfig.withBinds(Bind.parse(LOCAL_IMAGE_TARS_DIR + ":" + LOCAL_IMAGE_TARS_DIR), + Bind.parse(CI_IMAGE_TARS_DIR + ":" + CI_IMAGE_TARS_DIR)); }); return this; diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Images.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Images.java index ef76b8009f..8a775723d6 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Images.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Images.java @@ -19,6 +19,8 @@ import java.io.BufferedReader; import java.io.InputStreamReader; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.testcontainers.k3s.K3sContainer; /** @@ -26,6 +28,8 @@ */ public final class Images { + private static final Log LOG = LogFactory.getLog(Images.class); + private static final String BUSYBOX = "busybox"; private static final String BUSYBOX_TAR = BUSYBOX + ":" + busyboxVersion(); @@ -48,11 +52,11 @@ public final class Images { private static final String KAFKA = "confluentinc/confluent-local"; - private static final String KAFKA_TAR = KAFKA.replace('/', '-') + kafkaVersion(); + private static final String KAFKA_TAR = KAFKA.replace('/', '-') + ":" + kafkaVersion(); private static final String RABBITMQ = "rabbitmq"; - private static final String RABBITMQ_TAR = "rabbitmq"; + private static final String RABBITMQ_TAR = "rabbitmq" + ":" + rabbitMqVersion(); private Images() { @@ -80,34 +84,34 @@ public static String wiremockVersion() { public static void loadBusybox(K3sContainer container) { if (!imageAlreadyInK3s(container, BUSYBOX_TAR)) { - Commons.loadOrPullCommonTestImages(container, BUSYBOX_TAR, BUSYBOX, busyboxVersion()); + K3sImageLoader.loadOrPullCommonTestImages(container, BUSYBOX_TAR, BUSYBOX, busyboxVersion()); } } public static void loadWiremock(K3sContainer container) { if (!imageAlreadyInK3s(container, WIREMOCK_TAR)) { - Commons.loadOrPullCommonTestImages(container, WIREMOCK_TAR, WIREMOCK, wiremockVersion()); + K3sImageLoader.loadOrPullCommonTestImages(container, WIREMOCK_TAR, WIREMOCK, wiremockVersion()); } } public static void loadIstioCtl(K3sContainer container) { - Commons.loadOrPullCommonTestImages(container, ISTIOCTL_TAR, ISTIOCTL, istioVersion()); + K3sImageLoader.loadOrPullCommonTestImages(container, ISTIOCTL_TAR, ISTIOCTL, istioVersion()); } public static void loadIstioProxyV2(K3sContainer container) { - Commons.loadOrPullCommonTestImages(container, ISTIO_PROXY_V2_TAR, ISTIO_PROXY_V2, istioVersion()); + K3sImageLoader.loadOrPullCommonTestImages(container, ISTIO_PROXY_V2_TAR, ISTIO_PROXY_V2, istioVersion()); } public static void loadIstioPilot(K3sContainer container) { - Commons.loadOrPullCommonTestImages(container, ISTIO_PILOT_TAR, ISTIO_PILOT, istioVersion()); + K3sImageLoader.loadOrPullCommonTestImages(container, ISTIO_PILOT_TAR, ISTIO_PILOT, istioVersion()); } public static void loadKafka(K3sContainer container) { - Commons.loadOrPullCommonTestImages(container, KAFKA_TAR, KAFKA, kafkaVersion()); + K3sImageLoader.loadOrPullCommonTestImages(container, KAFKA_TAR, KAFKA, kafkaVersion()); } public static void loadRabbitmq(K3sContainer container) { - Commons.loadOrPullCommonTestImages(container, RABBITMQ_TAR, RABBITMQ, rabbitMqVersion()); + K3sImageLoader.loadOrPullCommonTestImages(container, RABBITMQ_TAR, RABBITMQ, rabbitMqVersion()); } private static boolean imageAlreadyInK3s(K3sContainer container, String tarName) { @@ -116,11 +120,11 @@ private static boolean imageAlreadyInK3s(K3sContainer container, String tarName) .getStdout() .contains(tarName); if (present) { - System.out.println("image : " + tarName + " already in k3s, skipping"); + LOG.info("image : " + tarName + " already in k3s, skipping"); return true; } else { - System.out.println("image : " + tarName + " not in k3s"); + LOG.info("image : " + tarName + " not in k3s"); return false; } } diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/K3sImageLoader.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/K3sImageLoader.java new file mode 100644 index 0000000000..09fa9f955c --- /dev/null +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/K3sImageLoader.java @@ -0,0 +1,280 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.integration.tests.commons; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import com.github.dockerjava.api.command.ListImagesCmd; +import com.github.dockerjava.api.command.PullImageCmd; +import com.github.dockerjava.api.command.SaveImageCmd; +import com.github.dockerjava.api.model.Image; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.k3s.K3sContainer; + +import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.pomVersion; +import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.CI_IMAGE_TARS_DIR; +import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.LOCAL_IMAGE_TARS_DIR; + +/** + * @author wind57 + */ +public final class K3sImageLoader { + + private static final Log LOG = LogFactory.getLog(K3sImageLoader.class); + + private static final String DOCKER_IO = "docker.io/"; + + private static final String DOCKER_IO_LIBRARY = DOCKER_IO + "library/"; + + private K3sImageLoader() { + + } + + public static void loadSpringCloudKubernetesImage(String imageNameWithoutTag, K3sContainer container) { + String tag = pomVersion(); + String imageNameWithTag = imageNameWithoutTag + ":" + tag; + + if (imageAlreadyInK3s(container, imageNameWithTag)) { + return; + } + + String tarFileName = imageNameWithTag + ".tar"; + Path ciTarPath = Paths.get(CI_IMAGE_TARS_DIR, tarFileName); + + // we are in github ci pipeline + if (Files.exists(ciTarPath)) { + loadCiImageTar(tarFileName, container); + return; + } + + createAndLoadLocalImageTar("springcloud/" + imageNameWithoutTag, tag, tarFileName, container); + } + + /** + * Ensures a common external test image is available inside K3s/containerd. It first + * checks whether the image is already present in K3s. If not, it tries to load it as + * a tar under '/tmp/docker/images'. If no matching tar is found, it pulls the image + * directly inside K3s using 'ctr images pull'. This is meant for shared test images + * such as busybox, wiremock, kafka, etc. + */ + public static void loadOrPullCommonTestImages(K3sContainer container, String tarName, String imageNameForDownload, + String imageVersion) { + + if (imageAlreadyInK3s(container, tarName)) { + return; + } + + File dockerImagesRootDir = Paths.get(CI_IMAGE_TARS_DIR).toFile(); + if (dockerImagesRootDir.exists() && dockerImagesRootDir.isDirectory()) { + File[] tars = dockerImagesRootDir.listFiles(); + if (tars != null && tars.length > 0) { + Optional found = Arrays.stream(tars) + .map(File::getName) + .filter(x -> x.contains(tarName)) + .findFirst(); + if (found.isPresent()) { + LOG.info("running in github actions, will load from : " + CI_IMAGE_TARS_DIR + " tar : " + + found.get()); + loadCiImageTar(found.get(), container); + return; + } + else { + LOG.info(tarName + " not found, resorting to pulling the image"); + } + } + else { + LOG.info("no tars found, will resort to pulling the image"); + } + } + else { + LOG.info("running outside github actions"); + } + + try { + LOG.info("pulling image inside k3s to avoid Docker save/ctr import compatibility issues"); + LOG.info("using : " + imageVersion + " for : " + imageNameForDownload); + pullImageInsideK3s(container, imageNameForDownload, imageVersion); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * validates that the provided image does exist in the local docker registry. This is + * only needed when running tests locally, in github actions images are already built + * by the pipeline. + */ + public static void validateImage(String image, K3sContainer container) { + try (ListImagesCmd listImagesCmd = container.getDockerClient().listImagesCmd()) { + List images = listImagesCmd.exec(); + images.stream() + .filter(x -> Arrays.stream(x.getRepoTags() == null ? new String[] {} : x.getRepoTags()) + .anyMatch(y -> y.contains(image))) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Image : " + image + " not build locally. " + + "You need to build it first, and then run the test")); + } + } + + public static void pullImage(String imageWithTag, String tarName, K3sContainer container) + throws InterruptedException { + + if (imageAlreadyInK3s(container, tarName)) { + return; + } + + try (PullImageCmd pullImageCmd = container.getDockerClient().pullImageCmd(imageWithTag)) { + pullImageCmd.start().awaitCompletion(); + } + } + + private static void loadCiImageTar(String tarFileName, K3sContainer container) { + loadImageTar(CI_IMAGE_TARS_DIR, tarFileName, container); + } + + private static void loadLocalImageTar(String tarFileName, K3sContainer container) { + loadImageTar(LOCAL_IMAGE_TARS_DIR, tarFileName, container); + } + + private static void loadImageTar(String imageTarsDir, String tarFileName, K3sContainer container) { + Awaitilities.awaitUntil(120, 1000, () -> { + Container.ExecResult result; + try { + result = container.execInContainer("ctr", "i", "import", imageTarsDir + "/" + tarFileName); + } + catch (Exception e) { + throw new RuntimeException(e); + } + boolean noErrors = result.getStderr() == null || result.getStderr().isEmpty(); + if (!noErrors) { + LOG.info("error is : " + result.getStderr()); + } + return noErrors; + }); + } + + /** + * Pull image directly inside the K3s container via ctr (containerd). This avoids + * Docker save + ctr import, which can fail with "content digest not found" for + * multi-platform or OCI images. If DOCKER_HUB_USERNAME and DOCKER_HUB_PASSWORD are + * set, they are passed as {@code --user username:password} for registry auth. + */ + private static void pullImageInsideK3s(K3sContainer container, String imageNameForDownload, String imageVersion) { + String fullImageRef = fullImageReference(imageNameForDownload, imageVersion); + + final String[] ctrArgs = buildCtrPullArgs(fullImageRef); + Awaitilities.awaitUntil(120, 1, () -> { + try { + Container.ExecResult result = container.execInContainer(ctrArgs); + boolean noErrors = result.getStderr() == null || result.getStderr().isEmpty(); + if (!noErrors) { + LOG.info("ctr pull stderr: " + result.getStderr()); + } + return noErrors; + } + catch (Exception e) { + throw new RuntimeException(e); + } + + }); + } + + /** + * Local fallback for Spring Cloud images: create a tar from the local Docker image if + * needed, then import it into K3s. + */ + private static void createAndLoadLocalImageTar(String imageNameWithoutTag, String tag, String tarFileName, + K3sContainer container) { + + String imageRef = imageNameWithoutTag + ":" + tag; + if (imageAlreadyInK3s(container, imageRef)) { + return; + } + + Path tarPath = Paths.get(LOCAL_IMAGE_TARS_DIR, tarFileName); + + if (Files.notExists(tarPath)) { + try (SaveImageCmd saveImageCmd = container.getDockerClient().saveImageCmd(imageNameWithoutTag); + InputStream imageStream = saveImageCmd.withTag(tag).exec()) { + Files.copy(imageStream, tarPath, StandardCopyOption.REPLACE_EXISTING); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + loadLocalImageTar(tarFileName, container); + } + + private static boolean imageAlreadyInK3s(K3sContainer container, String tarName) { + + if (tarName == null) { + return false; + } + + try { + boolean present = container.execInContainer("sh", "-c", "ctr images list | grep " + tarName) + .getStdout() + .contains(tarName); + if (present) { + LOG.info("image : " + tarName + " already in k3s, skipping"); + return true; + } + else { + LOG.info("image : " + tarName + " not in k3s"); + return false; + } + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static String[] buildCtrPullArgs(String fullImageRef) { + String username = System.getenv("DOCKER_HUB_USERNAME"); + String password = System.getenv("DOCKER_HUB_PASSWORD"); + if (username != null && !username.isBlank() && password != null && !password.isBlank()) { + LOG.info("pulling inside k3s with Docker Hub credentials: " + fullImageRef); + return new String[] { "ctr", "-n", "k8s.io", "images", "pull", "--user", username + ":" + password, + fullImageRef }; + } + LOG.info("pulling inside k3s: " + fullImageRef); + return new String[] { "ctr", "-n", "k8s.io", "images", "pull", fullImageRef }; + } + + private static String fullImageReference(String imageName, String imageVersion) { + String imageNameAndVersion = imageName + ":" + imageVersion; + if (imageName.contains("/")) { + return DOCKER_IO + imageNameAndVersion; + } + return DOCKER_IO_LIBRARY + imageNameAndVersion; + } + +} diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Fabric8ClientKubernetesFixture.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Fabric8ClientKubernetesFixture.java index ab5859a25a..6d8e432aaf 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Fabric8ClientKubernetesFixture.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Fabric8ClientKubernetesFixture.java @@ -41,13 +41,11 @@ import org.testcontainers.k3s.K3sContainer; import org.springframework.cloud.kubernetes.integration.tests.commons.Awaitilities; +import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; import org.springframework.cloud.kubernetes.integration.tests.commons.Images; +import org.springframework.cloud.kubernetes.integration.tests.commons.K3sImageLoader; import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.loadImage; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.pomVersion; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.pullImage; - /** * @author wind57 */ @@ -70,31 +68,29 @@ public Fabric8ClientKubernetesFixture(K3sContainer container) { * service. It creates the given resources as-well as waits for them to be created. * The delay check is intentionally not taken as an argument, so that it stays as * tight as possible, providing reasonable defaults. - * + * @param appendTag is false for common test images like busybox, wiremock. it is true + * for our images that we build from integration tests. */ - public void createAndWait(String namespace, String name, @Nullable Deployment deployment, @Nullable Service service, - boolean changeVersion) { + public void createAndWait(String namespace, @Nullable Deployment deployment, @Nullable Service service, + boolean appendTag) { try { if (deployment != null) { - String imageFromDeployment = deployment.getSpec() - .getTemplate() - .getSpec() - .getContainers() - .get(0) - .getImage(); - if (changeVersion) { - deployment.getSpec() + + if (appendTag) { + String imageFromDeployment = deployment.getSpec() .getTemplate() .getSpec() .getContainers() .get(0) - .setImage(imageFromDeployment + ":" + pomVersion()); - } - else { - String[] image = imageFromDeployment.split(":", 2); - pullImage(image[0], image[1], name, container); - loadImage(image[0], image[1], name, container); + .getImage(); + + String imageNameWithoutTag = imageFromDeployment.replace("docker.io/springcloud/", ""); + K3sImageLoader.loadSpringCloudKubernetesImage(imageNameWithoutTag, container); + + String imageWithVersion = imageFromDeployment + ":" + Commons.pomVersion(); + + deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setImage(imageWithVersion); } client.apps().deployments().inNamespace(namespace).resource(deployment).create(); @@ -123,7 +119,7 @@ public void busybox(String namespace, Phase phase) { Service service = Serialization.unmarshal(serviceStream, Service.class); if (phase.equals(Phase.CREATE)) { - createAndWait(namespace, "busybox", deployment, service, false); + createAndWait(namespace, deployment, service, false); } else if (phase.equals(Phase.DELETE)) { deleteAndWait(namespace, deployment, service); @@ -134,7 +130,7 @@ public void externalName(Phase phase) { InputStream serviceStream = inputStream("external-name-service/external-name-service.yaml"); Service service = Serialization.unmarshal(serviceStream, Service.class); if (Phase.CREATE.equals(phase)) { - createAndWait("default", null, null, service, false); + createAndWait("default", null, service, false); } else { deleteAndWait("default", null, service); @@ -288,7 +284,7 @@ public void istioCtl(String namespace, Phase phase) { istioctlDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).setImage(imageWithVersion); if (phase.equals(Phase.CREATE)) { - createAndWait(namespace, null, istioctlDeployment, null, false); + createAndWait(namespace, istioctlDeployment, null, false); } else { deleteAndWait(namespace, istioctlDeployment, null); @@ -351,7 +347,7 @@ public void wiremock(String namespace, Phase phase, boolean withNodePort) { if (phase.equals(Phase.CREATE)) { deployment.getMetadata().setNamespace(namespace); service.getMetadata().setNamespace(namespace); - createAndWait(namespace, "wiremock", deployment, service, false); + createAndWait(namespace, deployment, service, false); } else { deleteAndWait(namespace, deployment, service); @@ -368,7 +364,7 @@ public void configWatcher(Phase phase) { Service service = client.services().load(serviceStream).item(); if (phase.equals(Phase.CREATE)) { - createAndWait("default", deployment.getMetadata().getName(), deployment, service, true); + createAndWait("default", deployment, service, true); } else if (phase.equals(Phase.DELETE)) { deleteAndWait("default", deployment, service); diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/FabricClientIntegrationTestExtension.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/FabricClientIntegrationTestExtension.java index 6c805a193a..47e79acc65 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/FabricClientIntegrationTestExtension.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/FabricClientIntegrationTestExtension.java @@ -32,6 +32,7 @@ import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; import org.springframework.cloud.kubernetes.integration.tests.commons.Images; +import org.springframework.cloud.kubernetes.integration.tests.commons.K3sImageLoader; import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Fabric8ClientKubernetesFixture; import org.springframework.core.annotation.AnnotatedElementUtils; @@ -64,9 +65,9 @@ public void beforeAll(ExtensionContext context) { } // 2. external image presence - for (String image : scenario.withImages()) { - Commons.validateImage(image, container); - Commons.loadSpringCloudKubernetesImage(image, container); + for (String imageNameWithoutTag : scenario.withImages()) { + K3sImageLoader.validateImage(imageNameWithoutTag, container); + K3sImageLoader.loadSpringCloudKubernetesImage(imageNameWithoutTag, container); } // 3. deploy istio @@ -230,7 +231,7 @@ private void istioSetup(K3sContainer container, Fabric8ClientKubernetesFixture f Deployment deployment = Serialization.unmarshal(deploymentStream, Deployment.class); Service service = Serialization.unmarshal(serviceStream, Service.class); - fabric8KubernetesFixture.createAndWait("istio-test", null, deployment, service, true); + fabric8KubernetesFixture.createAndWait("istio-test", deployment, service, true); } catch (Exception e) { diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java index 5af4da9ee4..d5258e3bcb 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java @@ -27,6 +27,7 @@ import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; import org.springframework.cloud.kubernetes.integration.tests.commons.Images; +import org.springframework.cloud.kubernetes.integration.tests.commons.K3sImageLoader; import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; import org.springframework.cloud.kubernetes.integration.tests.commons.native_client.NativeClientKubernetesFixture; import org.springframework.core.annotation.AnnotatedElementUtils; @@ -58,9 +59,9 @@ public void beforeAll(ExtensionContext context) throws Exception { } // 2. external image presence - for (String image : scenario.withImages()) { - Commons.validateImage(image, container); - Commons.loadSpringCloudKubernetesImage(image, container); + for (String imageNameWithoutTag : scenario.withImages()) { + K3sImageLoader.validateImage(imageNameWithoutTag, container); + K3sImageLoader.loadSpringCloudKubernetesImage(imageNameWithoutTag, container); } // 3. set-up RBAC. @@ -110,11 +111,13 @@ public void beforeAll(ExtensionContext context) throws Exception { // 10. deploy kafka if (scenario.deployKafka()) { + Images.loadKafka(container); nativeClientKubernetesFixture.kafka(Phase.CREATE); } // 11. deploy rabbitMq if (scenario.deployRabbitMq()) { + Images.loadRabbitmq(container); nativeClientKubernetesFixture.rabbitMq(Phase.CREATE); } } diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java index 4b567c8f1d..bdcf122c9a 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java @@ -58,13 +58,11 @@ import org.testcontainers.k3s.K3sContainer; import org.springframework.cloud.kubernetes.integration.tests.commons.Awaitilities; +import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; import org.springframework.cloud.kubernetes.integration.tests.commons.Images; +import org.springframework.cloud.kubernetes.integration.tests.commons.K3sImageLoader; import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.loadImage; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.pomVersion; -import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.pullImage; - /** * @author wind57 */ @@ -106,31 +104,26 @@ public NativeClientKubernetesFixture(K3sContainer container) { * tight as possible, providing reasonable defaults. * */ - public void createAndWait(String namespace, String name, V1Deployment deployment, V1Service service, - boolean changeVersion) { + public void createAndWait(String namespace, V1Deployment deployment, V1Service service, boolean appendTag) { try { coreV1Api.createNamespacedService(namespace, service).execute(); if (deployment != null) { - String imageFromDeployment = deployment.getSpec() - .getTemplate() - .getSpec() - .getContainers() - .get(0) - .getImage(); - if (changeVersion) { - deployment.getSpec() + + if (appendTag) { + String imageFromDeployment = deployment.getSpec() .getTemplate() .getSpec() .getContainers() .get(0) - .setImage(imageFromDeployment + ":" + pomVersion()); - } - else { - String[] image = imageFromDeployment.split(":", 2); - pullImage(image[0], image[1], name, container); - loadImage(image[0], image[1], name, container); + .getImage(); + + String imageNameWithoutTag = imageFromDeployment.substring("docker.io/springcloud/".length()); + K3sImageLoader.loadSpringCloudKubernetesImage(imageNameWithoutTag, container); + + String imageWithVersion = imageFromDeployment + ":" + Commons.pomVersion(); + deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setImage(imageWithVersion); } appsV1Api.createNamespacedDeployment(namespace, deployment).execute(); @@ -268,13 +261,13 @@ public void deleteAndWait(String namespace, V1Deployment deployment, V1Service s public void busybox(String namespace, Phase phase) { V1Deployment deployment = yaml("busybox/deployment.yaml", V1Deployment.class); - String imageWithoutVersion = deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getImage(); - String imageWithVersion = imageWithoutVersion + ":" + Images.busyboxVersion(); - deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setImage(imageWithVersion); + String imageWithoutTag = deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getImage(); + String imageWithTag = imageWithoutTag + ":" + Images.busyboxVersion(); + deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setImage(imageWithTag); V1Service service = yaml("busybox/service.yaml", V1Service.class); if (phase.equals(Phase.CREATE)) { - createAndWait(namespace, "busybox", deployment, service, false); + createAndWait(namespace, deployment, service, false); } else if (phase.equals(Phase.DELETE)) { deleteAndWait(namespace, deployment, service); @@ -291,7 +284,7 @@ public void kafka(Phase phase) { V1Service service = yaml("kafka/kafka-service.yaml", V1Service.class); if (phase.equals(Phase.CREATE)) { - createAndWait("default", "kafka", deployment, service, false); + createAndWait("default", deployment, service, false); } else if (phase.equals(Phase.DELETE)) { deleteAndWait("default", deployment, service); @@ -308,7 +301,7 @@ public void rabbitMq(Phase phase) { V1Service service = yaml("rabbitmq/rabbitmq-service.yaml", V1Service.class); if (phase.equals(Phase.CREATE)) { - createAndWait("default", "rabbitmq", deployment, service, false); + createAndWait("default", deployment, service, false); } else if (phase.equals(Phase.DELETE)) { deleteAndWait("default", deployment, service); @@ -495,7 +488,7 @@ public void wiremock(String namespace, Phase phase, boolean withNodePort) { if (phase.equals(Phase.CREATE)) { deployment.getMetadata().setNamespace(namespace); service.getMetadata().setNamespace(namespace); - createAndWait(namespace, "wiremock", deployment, service, false); + createAndWait(namespace, deployment, service, false); } else { deleteAndWait(namespace, deployment, service); @@ -506,7 +499,7 @@ public void wiremock(String namespace, Phase phase, boolean withNodePort) { public void externalName(Phase phase) { V1Service service = yaml("external-name-service/external-name-service.yaml", V1Service.class); if (Phase.CREATE.equals(phase)) { - createAndWait("default", null, null, service, false); + createAndWait("default", null, service, false); } else { deleteAndWait("default", null, service); @@ -555,7 +548,7 @@ public void configWatcher(Phase phase, String refreshDelay, boolean reloadEnable deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars); if (phase.equals(Phase.CREATE)) { - createAndWait("default", deployment.getMetadata().getName(), deployment, service, true); + createAndWait("default", deployment, service, true); } else if (phase.equals(Phase.DELETE)) { deleteAndWait("default", deployment, service); @@ -567,7 +560,7 @@ public void discoveryServer(Phase phase) { V1Service service = yaml("discovery-server/discoveryserver-service.yaml", V1Service.class); if (phase == Phase.CREATE) { - createAndWait("default", null, deployment, service, true); + createAndWait("default", deployment, service, true); } else { deleteAndWait("default", null, service);