diff --git a/Makefile b/Makefile index 5985a0ad3..57748a7ed 100644 --- a/Makefile +++ b/Makefile @@ -204,7 +204,8 @@ else helm repo update syself KUBECONFIG=$(WORKER_CLUSTER_KUBECONFIG) helm upgrade --install ccm syself/ccm-hetzner --version 2.0.6 \ --namespace kube-system \ - --set privateNetwork.enabled=$(PRIVATE_NETWORK) + --set privateNetwork.enabled=$(PRIVATE_NETWORK) \ + --set-json 'extraEnvVars=[{"name":"HCLOUD_USE_HROBOT_PROVIDER_ID_FOR_BAREMETAL","value":"true"}]' @echo @echo 'run "kubectl --kubeconfig=$(WORKER_CLUSTER_KUBECONFIG) ..." to work with the new target cluster' @echo diff --git a/docs/caph/02-topics/05-baremetal/03-creating-workload-cluster.md b/docs/caph/02-topics/05-baremetal/03-creating-workload-cluster.md index 3b842e67e..ba83b2969 100644 --- a/docs/caph/02-topics/05-baremetal/03-creating-workload-cluster.md +++ b/docs/caph/02-topics/05-baremetal/03-creating-workload-cluster.md @@ -13,7 +13,10 @@ Secrets as of now are hardcoded given we are using a flavor which is essentially {% /callout %} -First you need to decide if you want to use the Syself CCM or the upstream HCloud CCM. +For bare metal, use the Syself CCM until an upstream release includes [PR fix robot name lookup +after stale cache miss](https://github.com/hetznercloud/hcloud-cloud-controller-manager/pull/1204). + +In the long run we (Syself) want to switch from our fork to the upstream CCM. The CCM is the "Cloud Controller" which runs in the workload-cluster. The most important tasks of the CCM are: @@ -21,9 +24,6 @@ The CCM is the "Cloud Controller" which runs in the workload-cluster. The most i machine (in mgt-cluster) is which Node (in wl-cluster). - Creates LoadBalancers -If you are unsure, use the HCloud CCM. In the long run we (Syself) want to switch from our fork to -the upstream CCM. - The CCM calls the Hetzner APIs. To authenticate, it reads the credentials from a secret. This secret has to be in the workload cluster, when the CCM runs in the workload cluster. CAPH creates the secret and syncs the credentials specified in the management cluster to the workload cluster. In our @@ -36,7 +36,7 @@ Important: CAPH and the CCM must both use the same ProviderID format for bare me - old: `hcloud://bm-NNNN` - new: `hrobot://NNNN` -The Syself CCM uses the old format by default. The HCloud CCM always uses the new format. +The upstream HCloud CCM uses the new format. If you use the new format, set the annotation `capi.syself.com/use-hrobot-provider-id-for-baremetal` to `"true"` on the `HetznerCluster`. Our default templates have this annotation set. @@ -108,47 +108,22 @@ This requires a secret containing access credentials to both Hetzner Robot and H {% /callout %} -If you want to use the HCloud CCM: +For bare metal, use the Syself CCM until an upstream release includes [PR fix robot name lookup +after stale cache miss](https://github.com/hetznercloud/hcloud-cloud-controller-manager/pull/1204). ```shell -helm repo add hcloud https://charts.hetzner.cloud -helm repo update hcloud +helm repo add syself https://charts.syself.com +helm repo update syself -helm upgrade --install ccm hcloud/hcloud-cloud-controller-manager \ +helm upgrade --install ccm syself/ccm-hetzner --version 2.0.6 \ --namespace kube-system \ - --set env.HCLOUD_TOKEN.valueFrom.secretKeyRef.name=hetzner \ - --set env.HCLOUD_TOKEN.valueFrom.secretKeyRef.key=hcloud \ - --set env.ROBOT_USER.valueFrom.secretKeyRef.name=hetzner \ - --set env.ROBOT_USER.valueFrom.secretKeyRef.key=robot-user \ - --set env.ROBOT_PASSWORD.valueFrom.secretKeyRef.name=hetzner \ - --set env.ROBOT_PASSWORD.valueFrom.secretKeyRef.key=robot-password \ - --set-json 'additionalTolerations=[{"key":"node.cluster.x-k8s.io/uninitialized","operator":"Exists","effect":"NoSchedule"},{"key":"node.cilium.io/agent-not-ready","operator":"Exists","effect":"NoSchedule"}]' \ - --set robot.enabled=true \ + --set-json 'extraEnvVars=[{"name":"HCLOUD_USE_HROBOT_PROVIDER_ID_FOR_BAREMETAL","value":"true"}]' \ --kubeconfig workload-kubeconfig ``` Be sure that the HetznerCluster has the annotation `capi.syself.com/use-hrobot-provider-id-for-baremetal: "true"`. -If you use a different `hetznerSecretRef.name` or different keys, then adjust the chart values -above accordingly. - ---- - -If you want to use the Syself CCM (not recommended for new clusters): - -```shell -helm repo add syself https://charts.syself.com -helm repo update syself - -$ helm upgrade --install ccm syself/ccm-hetzner --version 2.0.6 \ - --namespace kube-system \ - --kubeconfig workload-kubeconfig -``` - -Be sure that the HetznerCluster does not have the annotation -`capi.syself.com/use-hrobot-provider-id-for-baremetal`. - ### Installing CNI For CNI, let's deploy cilium in the workload cluster that will facilitate the networking in the cluster. diff --git a/hack/verify-generated-files.sh b/hack/verify-generated-files.sh index f530d3b1f..29d511382 100755 --- a/hack/verify-generated-files.sh +++ b/hack/verify-generated-files.sh @@ -27,6 +27,7 @@ if ! git diff --quiet || ! git diff --cached --quiet || [[ -n "$(git ls-files -- fi ( + export PATH="$(git rev-parse --show-toplevel)/hack/tools/bin:$PATH" make kubectl cd test/e2e HCLOUD_TOKEN=dummy_hcloud_token diff --git a/templates/cluster-templates/v1beta1/bases/hetzner-hetznerCluster.yaml b/templates/cluster-templates/v1beta1/bases/hetzner-hetznerCluster.yaml index 6039ec052..3dae7e035 100644 --- a/templates/cluster-templates/v1beta1/bases/hetzner-hetznerCluster.yaml +++ b/templates/cluster-templates/v1beta1/bases/hetzner-hetznerCluster.yaml @@ -26,6 +26,6 @@ spec: hetznerSecretRef: name: hcloud key: - hcloudToken: token + hcloudToken: hcloud hetznerRobotUser: robot-user hetznerRobotPassword: robot-password diff --git a/test/e2e/Makefile b/test/e2e/Makefile index e6969f840..a19b7614d 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -68,6 +68,7 @@ e2e-cilium-templates: e2e-ccm-templates: helm repo add hetzner-cloud https://charts.hetzner.cloud + helm repo add syself https://charts.syself.com helm template ccm hetzner-cloud/hcloud-cloud-controller-manager --version 1.30.1 \ --namespace kube-system \ @@ -89,27 +90,20 @@ e2e-ccm-templates: --set networking.network.valueFrom.secretKeyRef.name=hcloud \ --set networking.network.valueFrom.secretKeyRef.key=network > $(REPO_ROOT)/test/e2e/data/ccm/hcloud-ccm-network.yaml - helm template ccm hetzner-cloud/hcloud-cloud-controller-manager --version 1.30.1 \ + helm template ccm syself/ccm-hetzner --version 2.0.6 \ --namespace kube-system \ - --set image.repository=ghcr.io/syself/hcloud-cloud-controller-manager \ - --set-string env.HCLOUD_DEBUG.value=true \ - --set env.HCLOUD_TOKEN.valueFrom.secretKeyRef.name=hcloud \ - --set env.HCLOUD_TOKEN.valueFrom.secretKeyRef.key=token \ - --set env.ROBOT_USER.valueFrom.secretKeyRef.name=hcloud \ - --set env.ROBOT_USER.valueFrom.secretKeyRef.key=robot-user \ - --set env.ROBOT_PASSWORD.valueFrom.secretKeyRef.name=hcloud \ - --set env.ROBOT_PASSWORD.valueFrom.secretKeyRef.key=robot-password \ - --set-json 'additionalTolerations=[{"key":"node.cluster.x-k8s.io/uninitialized","operator":"Exists","effect":"NoSchedule"},{"key":"node.cilium.io/agent-not-ready","operator":"Exists","effect":"NoSchedule"}]' \ - --set args.route-reconciliation-period=null \ - --set args.feature-gates=null \ - --set robot.enabled=true \ - --set nameOverride=ccm-ccm-hetzner \ - --set networking.enabled=false > $(REPO_ROOT)/test/e2e/data/ccm/hcloud-ccm-hetzner.yaml + --set env.debug=true \ + --set pdb.enabled=false \ + --set privateNetwork.enabled=false \ + --set secret.name=hcloud \ + --set-json 'extraEnvVars=[{"name":"HCLOUD_USE_HROBOT_PROVIDER_ID_FOR_BAREMETAL","value":"true"}]' \ + --set-json 'tolerations=[{"key":"node.cluster.x-k8s.io/uninitialized","operator":"Exists","effect":"NoSchedule"},{"key":"node.cilium.io/agent-not-ready","operator":"Exists","effect":"NoSchedule"}]' \ + > $(REPO_ROOT)/test/e2e/data/ccm/syself-ccm-hetzner.yaml for file in \ $(REPO_ROOT)/test/e2e/data/ccm/hcloud-ccm.yaml \ $(REPO_ROOT)/test/e2e/data/ccm/hcloud-ccm-network.yaml \ - $(REPO_ROOT)/test/e2e/data/ccm/hcloud-ccm-hetzner.yaml; do \ + $(REPO_ROOT)/test/e2e/data/ccm/syself-ccm-hetzner.yaml; do \ tmp_file="$${file}.tmp"; \ { \ # Keep the generated-file marker split so boilerplate verification does not classify this Makefile as generated. \ diff --git a/test/e2e/config/hetzner.yaml b/test/e2e/config/hetzner.yaml index 8a49a1a8f..2ab03a1ac 100644 --- a/test/e2e/config/hetzner.yaml +++ b/test/e2e/config/hetzner.yaml @@ -158,7 +158,7 @@ variables: CILIUM: "./data/cni/cilium/values.yaml" CCM: "./data/ccm/hcloud-ccm.yaml" CCM_NETWORK: "./data/ccm/hcloud-ccm-network.yaml" - CCM_HETZNER: "./data/ccm/hcloud-ccm-hetzner.yaml" + CCM_HETZNER: "./data/ccm/syself-ccm-hetzner.yaml" # Conformance testing variables CONFORMANCE_WORKER_MACHINE_COUNT: 3 diff --git a/test/e2e/data/ccm/hcloud-ccm-hetzner.yaml b/test/e2e/data/ccm/hcloud-ccm-hetzner.yaml deleted file mode 100644 index e8c79788a..000000000 --- a/test/e2e/data/ccm/hcloud-ccm-hetzner.yaml +++ /dev/null @@ -1,183 +0,0 @@ -# Code generated by `make -C test/e2e e2e-ccm-templates`. DO NOT EDIT. ---- -# Source: hcloud-cloud-controller-manager/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ccm-ccm-hetzner - namespace: kube-system ---- -# Source: hcloud-cloud-controller-manager/templates/clusterrole.yml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: "system:ccm-ccm-hetzner" -rules: - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - "*" - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - "" - resources: - - services - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - services/status - verbs: - - patch - - update - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - update - - watch - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - list - - watch - - update ---- -# Source: hcloud-cloud-controller-manager/templates/clusterrolebinding.yaml -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - # The prefix ":restricted" originates from removing the cluster-admin role from HCCM. - # Renaming the ClusterRoleBinding makes the migration easier for users. - name: "system:ccm-ccm-hetzner:restricted" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: "system:ccm-ccm-hetzner" -subjects: - - kind: ServiceAccount - name: ccm-ccm-hetzner - namespace: kube-system ---- -# Source: hcloud-cloud-controller-manager/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ccm-ccm-hetzner - namespace: kube-system -spec: - replicas: 1 - revisionHistoryLimit: 2 - selector: - matchLabels: - app.kubernetes.io/instance: 'ccm' - app.kubernetes.io/name: 'ccm-ccm-hetzner' - template: - metadata: - labels: - app.kubernetes.io/instance: 'ccm' - app.kubernetes.io/name: 'ccm-ccm-hetzner' - spec: - serviceAccountName: ccm-ccm-hetzner - dnsPolicy: Default - tolerations: - # Allow HCCM itself to schedule on nodes that have not yet been initialized by HCCM. - - key: "node.cloudprovider.kubernetes.io/uninitialized" - value: "true" - effect: "NoSchedule" - - key: "CriticalAddonsOnly" - operator: "Exists" - - # Allow HCCM to schedule on control plane nodes. - - key: "node-role.kubernetes.io/master" - effect: NoSchedule - operator: Exists - - key: "node-role.kubernetes.io/control-plane" - effect: NoSchedule - operator: Exists - - - key: "node.kubernetes.io/not-ready" - effect: "NoExecute" - - - effect: NoSchedule - key: node.cluster.x-k8s.io/uninitialized - operator: Exists - - effect: NoSchedule - key: node.cilium.io/agent-not-ready - operator: Exists - containers: - - name: hcloud-cloud-controller-manager - args: - - "--allow-untagged-cloud" - - "--cloud-provider=hcloud" - - "--webhook-secure-port=0" - - "--leader-elect=false" - env: - - name: HCLOUD_DEBUG - value: "true" - - name: HCLOUD_TOKEN - valueFrom: - secretKeyRef: - key: token - name: hcloud - - name: ROBOT_PASSWORD - valueFrom: - secretKeyRef: - key: robot-password - name: hcloud - optional: true - - name: ROBOT_USER - valueFrom: - secretKeyRef: - key: robot-user - name: hcloud - optional: true - - name: ROBOT_ENABLED - value: "true" - image: ghcr.io/syself/hcloud-cloud-controller-manager:v1.30.1 # x-releaser-pleaser-version - ports: - - name: metrics - containerPort: 8233 - resources: - requests: - cpu: 100m - memory: 50Mi - priorityClassName: "system-cluster-critical" diff --git a/test/e2e/data/ccm/syself-ccm-hetzner.yaml b/test/e2e/data/ccm/syself-ccm-hetzner.yaml new file mode 100644 index 000000000..7346d9657 --- /dev/null +++ b/test/e2e/data/ccm/syself-ccm-hetzner.yaml @@ -0,0 +1,123 @@ +# Code generated by `make -C test/e2e e2e-ccm-templates`. DO NOT EDIT. +--- +# Source: ccm-hetzner/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ccm-ccm-hetzner + namespace: kube-system + labels: + helm.sh/chart: ccm-hetzner-2.0.6 + app: ccm + app.kubernetes.io/name: ccm-hetzner + app.kubernetes.io/instance: ccm + app.kubernetes.io/version: "v2.0.6" + app.kubernetes.io/managed-by: Helm +--- +# Source: ccm-hetzner/templates/serviceaccount.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ccm-ccm-hetzner + namespace: kube-system + labels: + helm.sh/chart: ccm-hetzner-2.0.6 + app: ccm + app.kubernetes.io/name: ccm-hetzner + app.kubernetes.io/instance: ccm + app.kubernetes.io/version: "v2.0.6" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: ccm-ccm-hetzner + namespace: kube-system +--- +# Source: ccm-hetzner/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ccm-ccm-hetzner + namespace: kube-system + labels: + helm.sh/chart: ccm-hetzner-2.0.6 + app: ccm + app.kubernetes.io/name: ccm-hetzner + app.kubernetes.io/instance: ccm + app.kubernetes.io/version: "v2.0.6" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + revisionHistoryLimit: 5 + selector: + matchLabels: + app.kubernetes.io/name: ccm-hetzner + app.kubernetes.io/instance: ccm + template: + metadata: + labels: + app.kubernetes.io/name: ccm-hetzner + app.kubernetes.io/instance: ccm + spec: + dnsPolicy: Default + serviceAccountName: ccm-ccm-hetzner + securityContext: + {} + volumes: + - name: hetzner-secret + secret: + secretName: hcloud + tolerations: + # Introduced with CAPI v1.4, more info: https://cluster-api.sigs.k8s.io/developer/providers/bootstrap.html#taint-nodes-at-creation + - key: "node.cluster.x-k8s.io/uninitialized" + effect: "NoSchedule" + # Allow HCCM itself to schedule on nodes that have not yet been initialized by HCCM. + - key: "node.cloudprovider.kubernetes.io/uninitialized" + value: "true" + effect: "NoSchedule" + - key: "CriticalAddonsOnly" + operator: "Exists" + # cloud controller manages should be able to run on masters + - key: "node-role.kubernetes.io/master" + effect: NoSchedule + operator: Exists + - key: "node-role.kubernetes.io/control-plane" + effect: NoSchedule + operator: Exists + - key: "node.kubernetes.io/not-ready" + effect: "NoSchedule" + - effect: NoSchedule + key: node.cluster.x-k8s.io/uninitialized + operator: Exists + - effect: NoSchedule + key: node.cilium.io/agent-not-ready + operator: Exists + containers: + - name: ccm-hetzner + securityContext: + {} + image: "ghcr.io/syself/hetzner-cloud-controller-manager:v2.0.6" + imagePullPolicy: Always + command: + - "/bin/hetzner-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=true" + - "--allow-untagged-cloud" + resources: + requests: + cpu: 100m + memory: 50Mi + volumeMounts: + - mountPath: /etc/hetzner-secret + name: hetzner-secret + readOnly: true + env: + - name: HCLOUD_DEBUG + value: "true" + - name: HCLOUD_LOAD_BALANCERS_ENABLED + value: "true" + - name: HCLOUD_USE_HROBOT_PROVIDER_ID_FOR_BAREMETAL + value: "true" diff --git a/test/e2e/data/infrastructure-hetzner/v1beta1/bases/hetzner-secret.yaml b/test/e2e/data/infrastructure-hetzner/v1beta1/bases/hetzner-secret.yaml index 822761740..46ff2bd76 100644 --- a/test/e2e/data/infrastructure-hetzner/v1beta1/bases/hetzner-secret.yaml +++ b/test/e2e/data/infrastructure-hetzner/v1beta1/bases/hetzner-secret.yaml @@ -1,5 +1,7 @@ apiVersion: v1 data: + hcloud: | + hcloud_secret_placeholder token: | hcloud_secret_placeholder robot-user: | diff --git a/test/e2e/data/infrastructure-hetzner/v1beta1/bases/hetznerbaremetalhosts-raid.yaml b/test/e2e/data/infrastructure-hetzner/v1beta1/bases/hetznerbaremetalhosts-raid.yaml index 49e7d352a..6ffe0981f 100644 --- a/test/e2e/data/infrastructure-hetzner/v1beta1/bases/hetznerbaremetalhosts-raid.yaml +++ b/test/e2e/data/infrastructure-hetzner/v1beta1/bases/hetznerbaremetalhosts-raid.yaml @@ -12,6 +12,7 @@ spec: maintenanceMode: false description: Test Machine 2 --- +# 2026-04-09: Disabled after CI hit repeated rescue-mode reboot timeouts. apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: HetznerBareMetalHost metadata: @@ -23,5 +24,5 @@ spec: wwn: - "0x500a07511756c36d" - "0x500a075117101996" - maintenanceMode: false + maintenanceMode: true description: Test Machine 6 diff --git a/test/e2e/data/infrastructure-hetzner/v1beta1/bases/hetznerbaremetalhosts.yaml b/test/e2e/data/infrastructure-hetzner/v1beta1/bases/hetznerbaremetalhosts.yaml index ef3d12fe7..680989f32 100644 --- a/test/e2e/data/infrastructure-hetzner/v1beta1/bases/hetznerbaremetalhosts.yaml +++ b/test/e2e/data/infrastructure-hetzner/v1beta1/bases/hetznerbaremetalhosts.yaml @@ -71,7 +71,7 @@ spec: maintenanceMode: true #### < --- set to false, when ok. description: Test Machine 1751550 --- -# 2026-03-12: Support fixed hardware issue. Now working. +# 2026-04-09: Disabled after CI hit repeated rescue-mode reboot timeouts. apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: HetznerBareMetalHost metadata: @@ -83,5 +83,5 @@ spec: serverID: 1763731 # ip: 144.76.101.50 rootDeviceHints: wwn: "0x500a07511756c36d" - maintenanceMode: false + maintenanceMode: true description: Test Machine 1751550