e2e: use port-forward instead of hostPort for registry access

Replace the localhost:30000 hostPort-based registry access with
Kubernetes port-forwarding. This makes the test runner work regardless
of the cluster network topology (not just kind with extraPortMappings).

- Remove port 30000 extraPortMappings from kind configs
- Add PortForward() to the registry package using SPDY
- Use port-forward in e2e steps and extension-developer-e2e
- Remove LOCAL_REGISTRY_HOST env var (no longer needed)
- Keep NodePort service for containerd on kind nodes (hosts.toml)

Author: joelanford

go.mod

@@ -90,6 +90,7 @@ require (
 	github.com/containerd/typeurl/v2 v2.2.3 // indirect
 	github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
 	github.com/containers/ocicrypt v1.2.1 // indirect
+	github.com/creack/pty v1.1.24 // indirect
 	github.com/cucumber/gherkin/go/v26 v26.2.0 // indirect
 	github.com/cucumber/messages/go/v21 v21.0.1 // indirect
 	github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 // indirect

go.sum

@@ -89,8 +89,8 @@ github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5z
 github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
-github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
-github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
+github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
+github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
 github.com/cucumber/gherkin/go/v26 v26.2.0 h1:EgIjePLWiPeslwIWmNQ3XHcypPsWAHoMCz/YEBKP4GI=
 github.com/cucumber/gherkin/go/v26 v26.2.0/go.mod h1:t2GAPnB8maCT4lkHL99BDCVNzCh1d7dBhCLt150Nr/0=
 github.com/cucumber/godog v0.15.1 h1:rb/6oHDdvVZKS66hrhpjFQFHjthFSrQBCOI1LwshNTI=

hack/kind-config/containerd/certs.d/docker-registry.operator-controller-e2e.svc.cluster.local:5000/hosts.toml

@@ -20,9 +20,6 @@ nodes:
           kubeletExtraArgs:
             node-labels: "ingress-ready=true"
           taints: []
-    extraMounts:
-      - hostPath: ./hack/kind-config/containerd/certs.d
-        containerPath: /etc/containerd/certs.d
   - role: control-plane
     kubeadmConfigPatches:
       - |
@@ -31,10 +28,3 @@ nodes:
           kubeletExtraArgs:
             node-labels: "ingress-ready=true"
           taints: []
-    extraMounts:
-      - hostPath: ./hack/kind-config/containerd/certs.d
-        containerPath: /etc/containerd/certs.d
-containerdConfigPatches:
-  - |-
-    [plugins."io.containerd.grpc.v1.cri".registry]
-      config_path = "/etc/containerd/certs.d"

hack/kind-config/containerd/certs.d/go.mod

@@ -14,10 +14,3 @@ nodes:
         apiServer:
             extraArgs:
               enable-admission-plugins: OwnerReferencesPermissionEnforcement
-    extraMounts:
-      - hostPath: ./hack/kind-config/containerd/certs.d
-        containerPath: /etc/containerd/certs.d
-containerdConfigPatches:
-  - |-    
-    [plugins."io.containerd.grpc.v1.cri".registry]
-      config_path = "/etc/containerd/certs.d"

kind-config/kind-config-2node.yaml

@@ -8,6 +8,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
@@ -49,7 +52,6 @@ func TestMain(m *testing.M) {
 	}
 
 	// Set env vars for setup.sh — single source of truth
-	os.Setenv("LOCAL_REGISTRY_HOST", localAddr)
 	os.Setenv("CATALOG_TAG", catalogTag)
 	os.Setenv("REG_PKG_NAME", regPkgName)
 
@@ -60,9 +62,56 @@ func TestMain(m *testing.M) {
 		panic(fmt.Sprintf("failed to run setup.sh: %v", err))
 	}
 
+	// Push images via crane through the port-forward.
+	// setup.sh tags images with CLUSTER_REGISTRY_HOST. We export them
+	// from the container runtime via "save" and push through the
+	// port-forward with crane.
+	containerRuntime := os.Getenv("CONTAINER_RUNTIME")
+	bundlePath := "bundles/registry-v1/registry-bundle:v0.0.1"
+	for _, path := range []string{bundlePath, catalogTag} {
+		srcRef := fmt.Sprintf("%s/%s", clusterRegistryHost, path)
+		pushRef := fmt.Sprintf("%s/%s", localAddr, path)
+		if err := saveAndPush(containerRuntime, srcRef, pushRef); err != nil {
+			panic(fmt.Sprintf("failed to push image %s: %v", pushRef, err))
+		}
+	}
+
 	os.Exit(m.Run())
 }
 
+// saveAndPush exports an image from the container runtime to a temp file,
+// then loads and pushes it to the registry via crane.
+func saveAndPush(containerRuntime, srcRef, pushRef string) error {
+	f, err := os.CreateTemp("", "image-*.tar")
+	if err != nil {
+		return fmt.Errorf("failed to create temp file: %w", err)
+	}
+	defer os.Remove(f.Name())
+	defer f.Close()
+
+	saveCmd := exec.Command(containerRuntime, "save", srcRef) //nolint:gosec
+	saveCmd.Stdout = f
+	if err := saveCmd.Run(); err != nil {
+		return fmt.Errorf("failed to save image %s: %w", srcRef, err)
+	}
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("failed to close temp file: %w", err)
+	}
+
+	tag, err := name.NewTag(srcRef)
+	if err != nil {
+		return fmt.Errorf("failed to parse tag %s: %w", srcRef, err)
+	}
+	img, err := tarball.ImageFromPath(f.Name(), &tag)
+	if err != nil {
+		return fmt.Errorf("failed to load image %s from tarball: %w", srcRef, err)
+	}
+	if err := crane.Push(img, pushRef, crane.Insecure); err != nil {
+		return fmt.Errorf("failed to push image %s: %w", pushRef, err)
+	}
+	return nil
+}
+
 func TestExtensionDeveloper(t *testing.T) {
 	t.Parallel()
 	cfg := ctrl.GetConfigOrDie()

kind-config/kind-config.yaml

@@ -8,12 +8,11 @@ help="setup.sh is used to build extensions using the operator-sdk and
 build the image + bundle image, and create a FBC image for the
 following bundle formats:
 - registry+v1
-This script will ensure that all images built are loaded onto
-a KinD cluster with the name specified in the arguments.
+Images are built and tagged locally; pushing to the registry is
+handled by the Go test code via crane + port-forward.
 The following environment variables are required for configuring this script:
 - \$OPERATOR_SDK - path to the operator-sdk binary.
 - \$CONTAINER_RUNTIME - container runtime to use (e.g. docker, podman).
-- \$LOCAL_REGISTRY_HOST - registry address accessible from the test process.
 - \$CLUSTER_REGISTRY_HOST - registry address accessible from inside the cluster.
 - \$CATALOG_TAG - OCI tag for the catalog image (e.g. e2e/test-catalog:v1).
 - \$REG_PKG_NAME - the name of the package for the extension.
@@ -26,7 +25,7 @@ Usage:
 # Input validation
 ########################################
 
-for var in OPERATOR_SDK CONTAINER_RUNTIME LOCAL_REGISTRY_HOST CLUSTER_REGISTRY_HOST CATALOG_TAG REG_PKG_NAME; do
+for var in OPERATOR_SDK CONTAINER_RUNTIME CLUSTER_REGISTRY_HOST CATALOG_TAG REG_PKG_NAME; do
   if [[ -z "${!var:-}" ]]; then
     echo "\$$var is required to be set"
     echo "${help}"
@@ -49,25 +48,13 @@ mkdir -p "${REG_DIR}"
 
 operator_sdk="${OPERATOR_SDK}"
 container_tool="${CONTAINER_RUNTIME}"
-# The path we use to push the image from _outside_ the cluster
-local_registry_host="${LOCAL_REGISTRY_HOST}"
-# The path we use _inside_ the cluster
 cluster_registry_host="${CLUSTER_REGISTRY_HOST}"
 
-tls_flag=""
-if [[ "$container_tool" == "podman" ]]; then
-  echo "Using podman container runtime; adding tls disable flag"
-  tls_flag="--tls-verify=false"
-fi
-
-catalog_push_tag="${local_registry_host}/${CATALOG_TAG}"
+catalog_tag="${cluster_registry_host}/${CATALOG_TAG}"
 reg_pkg_name="${REG_PKG_NAME}"
 
 reg_img="${DOMAIN}/registry:v0.0.1"
-reg_bundle_path="bundles/registry-v1/registry-bundle:v0.0.1"
-
-reg_bundle_img="${cluster_registry_host}/${reg_bundle_path}"
-reg_bundle_push_tag="${local_registry_host}/${reg_bundle_path}"
+reg_bundle_img="${cluster_registry_host}/bundles/registry-v1/registry-bundle:v0.0.1"
 
 ########################################
 # Create the registry+v1 based extension
@@ -98,10 +85,14 @@ reg_bundle_push_tag="${local_registry_host}/${reg_bundle_path}"
   make docker-build IMG="${reg_img}" && \
   sed -i -e 's/$(OPERATOR_SDK) generate kustomize manifests -q/$(OPERATOR_SDK) generate kustomize manifests -q --interactive=false/g' Makefile && \
   make bundle IMG="${reg_img}" VERSION=0.0.1 && \
-  make bundle-build BUNDLE_IMG="${reg_bundle_push_tag}"
-  ${container_tool} push ${reg_bundle_push_tag} ${tls_flag}
+  make bundle-build BUNDLE_IMG="${reg_bundle_img}"
 )
 
+# Push is handled by the Go test via crane + port-forward,
+# because docker push goes through the Docker daemon which
+# may be in a different network context (e.g. colima VM).
+
+
 ###############################
 # Create the FBC that contains
 # the registry+v1 extensions
@@ -146,5 +137,6 @@ cat <<EOF > "${TMP_ROOT}"/catalog/index.yaml
 }
 EOF
 
-${container_tool} build --provenance=false -f "${TMP_ROOT}/catalog.Dockerfile" -t "${catalog_push_tag}" "${TMP_ROOT}/"
-${container_tool} push ${catalog_push_tag} ${tls_flag}
+${container_tool} build --provenance=false -f "${TMP_ROOT}/catalog.Dockerfile" -t "${catalog_tag}" "${TMP_ROOT}/"
+
+# Push is handled by the Go test via crane + port-forward.

test/extension-developer-e2e/extension_developer_test.go

@@ -32,7 +32,6 @@ import (
 const (
 	DefaultNamespace = "operator-controller-e2e"
 	DefaultName      = "docker-registry"
-	nodePort         = int32(30000)
 )
 
 // Deploy ensures the image registry namespace, TLS certificate, deployment,
@@ -123,18 +122,14 @@ func Deploy(ctx context.Context, cfg *rest.Config, namespace, name string) error
 		return fmt.Errorf("failed to apply deployment: %w", err)
 	}
 
-	// Apply service — NodePort so that containerd on the kind node can
-	// reach the registry via localhost:30000 (configured in hosts.toml).
-	// Test runners access the registry via port-forward instead.
+	// Apply service
 	svc := corev1ac.Service(name, namespace).
 		WithSpec(corev1ac.ServiceSpec().
 			WithSelector(map[string]string{"app": "registry"}).
-			WithType(corev1.ServiceTypeNodePort).
 			WithPorts(corev1ac.ServicePort().
 				WithName("http").
 				WithPort(5000).
-				WithTargetPort(intstr.FromInt32(5000)).
-				WithNodePort(nodePort),
+				WithTargetPort(intstr.FromInt32(5000)),
 			),
 		)
 	if err := c.Apply(ctx, svc, fieldOwner, client.ForceOwnership); err != nil {
@@ -167,21 +162,26 @@ func Deploy(ctx context.Context, cfg *rest.Config, namespace, name string) error
 // PortForward establishes a port-forward to the registry pod and returns
 // the local address (e.g. "localhost:12345") that can be used to push images.
 // The returned stop function should be called to clean up the port-forward.
-func PortForward(ctx context.Context, cfg *rest.Config, namespace, name string) (localAddr string, stop func(), err error) {
+func PortForward(ctx context.Context, cfg *rest.Config, namespace, name string) (string, func(), error) {
 	clientset, err := kubernetes.NewForConfig(cfg)
 	if err != nil {
 		return "", nil, fmt.Errorf("failed to create kubernetes client: %w", err)
 	}
 
+	deploy, err := clientset.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		return "", nil, fmt.Errorf("failed to get deployment %s/%s: %w", namespace, name, err)
+	}
+	labelSelector := metav1.FormatLabelSelector(deploy.Spec.Selector)
 	pods, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
-		LabelSelector: "app=registry",
+		LabelSelector: labelSelector,
 		FieldSelector: "status.phase=Running",
 	})
 	if err != nil {
-		return "", nil, fmt.Errorf("failed to list registry pods: %w", err)
+		return "", nil, fmt.Errorf("failed to list pods for deployment %s: %w", name, err)
 	}
 	if len(pods.Items) == 0 {
-		return "", nil, fmt.Errorf("no running registry pods found in %s", namespace)
+		return "", nil, fmt.Errorf("no running pods found for deployment %s/%s", namespace, name)
 	}
 	podName := pods.Items[0].Name