feature: debug builds for the host operator

In order to be able to debug the operator live in the local cluster, we
need the binary to be executed with Delve instead, and we need it to be
built without any code optimizations or minimization so that the
breakpoints work.

For that purpose, a new Dockerfile is added which can build the image
that way, and the corresponding Make targets make it easy to kick off
the whole process with a single command.

The Makefile targets also enable the "toolchain-e2e" repository to
easily build the "debug" image.

Also, when Developer Sandbox is deployed locally, usually the
registration service gets managed by the "ToolChain config controller",
which launches it with a specific command. In order to be able to launch
the registration service with Delve, a few minor modifications are made
so that that launch command can be overridden.

SANDBOX-1561

Author: MikelAlejoBR

build/Dockerfile

@@ -1,7 +1,12 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
+# Define the default build as "prod", so that in the case that no build
+# arguments are provided, we build the optimized image.
+ARG BUILD_TYPE=prod
 
-LABEL maintainer "KubeSaw <devsandbox@redhat.com>"
-LABEL author "KubeSaw <devsandbox@redhat.com>"
+# Default production build setps for the operator.
+FROM registry.access.redhat.com/ubi8/ubi-minimal:latest AS prod-build
+
+LABEL maintainer="KubeSaw <devsandbox@redhat.com>"
+LABEL author="KubeSaw <devsandbox@redhat.com>"
 
 ENV OPERATOR=/usr/local/bin/host-operator \
     USER_UID=1001 \
@@ -17,3 +22,42 @@ RUN  /usr/local/bin/user_setup
 ENTRYPOINT ["/usr/local/bin/entrypoint"]
 
 USER ${USER_UID}
+
+# A throwaway intermediate path which compiles Delve with the same Golang
+# version as the host computer. That ensures no discrepancies in the code
+# when setting up breakpoints when debgugging.
+FROM registry.access.redhat.com/ubi8/ubi-minimal:latest AS debug-initial-build
+
+# In this build path, the Golang version must be provided as a build argument,
+# which will ensure that Delve gets built with the same version the operator's
+# binary gets built with, to avoid any discrepancies.
+ARG GOLANG_VERSION
+
+# Install the tools to be able to download and extract Golang.
+RUN microdnf install --assumeyes \
+    curl \
+    gzip \
+    tar \
+    && microdnf clean all
+
+# Download and extract Golang.
+RUN curl --location --output "/tmp/${GOLANG_VERSION}.linux-amd64.tar.gz" "https://go.dev/dl/${GOLANG_VERSION}.linux-amd64.tar.gz" \
+    && tar --directory /usr/local --extract --file "/tmp/${GOLANG_VERSION}.linux-amd64.tar.gz" \
+    && rm --force "/tmp/${GOLANG_VERSION}.linux-amd64.tar.gz"
+
+# Put Golang in the path so that we can compile Delve with it.
+ENV PATH=$PATH:/usr/local/go/bin
+
+# Build Delve and leave it in a temporary directory.
+RUN GOBIN=/tmp/bin go install github.com/go-delve/delve/cmd/dlv@latest
+
+# In the case that the "BUILD_TYPE=debug" build argument is provided, we
+# "attempt to copy" the Delve executable into the production image. Since that
+# creates a dependency with the "debug-initial-build", that path is executed
+# by building the Delve executable.
+FROM prod-build AS debug-build
+COPY --from=debug-initial-build /tmp/bin/dlv /usr/local/bin/dlv
+
+# When the no build argument is provided, we simply fall back to the
+# production build which comes with the optimized and minimized image.
+FROM ${BUILD_TYPE}-build AS final-build

build/bin/entrypoint

@@ -9,4 +9,4 @@ if ! whoami &>/dev/null; then
   fi
 fi
 
-exec ${OPERATOR} $@
+exec ${OPERATOR} "$@"

controllers/toolchainconfig/env.go

@@ -1,3 +1,6 @@
 package toolchainconfig
 
-const RegistrationServiceImageEnvKey = "REGISTRATION_SERVICE_IMAGE"
+const (
+	RegistrationServiceImageEnvKey   = "REGISTRATION_SERVICE_IMAGE"
+	RegistrationServiceCommandEnvKey = "REGISTRATION_SERVICE_COMMAND"
+)

controllers/toolchainconfig/toolchainconfig_controller.go

@@ -151,6 +151,16 @@ func getVars(namespace string, cfg ToolchainConfig) templateVars {
 	vars["IMAGE"] = image
 	vars.addIfNotEmpty("NAMESPACE", namespace)
 	vars.addIfNotEmpty("REPLICAS", fmt.Sprint(cfg.RegistrationService().Replicas()))
+
+	// Allow overriding the registration service's command via an environment
+	// variable.
+	command := os.Getenv(RegistrationServiceCommandEnvKey)
+	if command != "" {
+		vars["COMMAND"] = command
+	} else {
+		vars["COMMAND"] = `["registration-service"]`
+	}
+
 	return vars
 }
 

deploy/templates/registration-service/registration-service.yaml

@@ -96,13 +96,12 @@ objects:
               image: ${IMAGE}
               ports:
                 - containerPort: 8080 # registration service
-                - containerPort: 8081 # proxy 
+                - containerPort: 8081 # proxy
                 - containerPort: 8082 # proxy metrics
                   name: metrics
                 - containerPort: 8083 # registration service metrics
                   name: regsvc-metrics
-              command:
-                - registration-service
+              command: ${{COMMAND}}
               imagePullPolicy: IfNotPresent
               livenessProbe:
                 failureThreshold: 3
@@ -141,7 +140,7 @@ objects:
                 requests:
                   cpu: "50m"
                   memory: "100M"
-  
+
   # route for the registration service
   - kind: Route
     apiVersion: v1
@@ -202,7 +201,7 @@ objects:
         run: registration-service
       type: ClusterIP
       sessionAffinity: null
-  
+
   # route for the proxy
   - kind: Route
     apiVersion: v1
@@ -224,7 +223,7 @@ objects:
       tls:
         termination: edge
       wildcardPolicy: None
- 
+
   # service associated with the proxy route
   - kind: Service
     apiVersion: v1
@@ -244,7 +243,7 @@ objects:
         run: registration-service
       type: ClusterIP
       sessionAffinity: null
-  
+
   # internal service for the proxy, used by Prometheus to scrape the metrics
   - kind: Service
     apiVersion: v1
@@ -271,3 +270,5 @@ parameters:
     value: quay.io/openshiftio/codeready-toolchain/registration-service:latest
   - name: REPLICAS
     value: '3'
+  - name: COMMAND
+    value: '["registration-service"]'

make/go.mk

@@ -20,6 +20,19 @@ $(OUT_DIR)/operator:
 		-o $(OUT_DIR)/bin/host-operator \
 		./cmd/main.go
 
+.PHONY: build-debug
+## Build the operator without the optimizations and the inlining, to enable
+## remote debugging via Delve.
+build-debug: generate
+	@echo "building host-operator in ${GO_PACKAGE_PATH}"
+	$(Q)go version
+	$(Q)CGO_ENABLED=0 GOARCH=${goarch} GOOS=linux \
+		go build ${V_FLAG} \
+		-gcflags "all=-N -l" \
+		-ldflags "-X ${GO_PACKAGE_PATH}/version.Commit=${GIT_COMMIT_ID} -X ${GO_PACKAGE_PATH}/version.BuildTime=${BUILD_TIME}" \
+		-o $(OUT_DIR)/bin/host-operator \
+		./cmd/main.go
+
 .PHONY: vendor
 vendor:
 	$(Q)go mod vendor

make/podman.mk

@@ -10,11 +10,21 @@ IMAGE_PLATFORM ?= linux/amd64
 podman-image: build
 	$(Q)podman build --platform ${IMAGE_PLATFORM} -f build/Dockerfile -t ${IMAGE} .
 
+## Build the operator's image with Delve on it so that it is ready to attach a
+## debugger to it.
+podman-image-debug: build-debug
+	$(Q) podman build --platform "${IMAGE_PLATFORM}" --build-arg BUILD_TYPE='debug' --build-arg GOLANG_VERSION="$$(go version | awk '{print $$3}')" --file build/Dockerfile.debug --tag "${IMAGE}" .
+
 .PHONY: podman-push
 ## Push the binary image to quay.io registry
 podman-push: check-namespace podman-image
 	$(Q)podman push ${IMAGE}
 
+.PHONY: podman-push-debug
+## Push the image with the debugger in it to the repository.
+podman-push-debug: check-namespace podman-image-debug
+	$(Q)podman push ${IMAGE}
+
 .PHONY: check-namespace
 check-namespace:
 ifeq ($(QUAY_NAMESPACE),${GO_PACKAGE_ORG_NAME})