Use k3s with containerd (kubernetes#2529)

Seems like in 1.29 there are issues with cri-dockerd. There's no reason
to use Docker anymore and this commit makes sure k3s uses containerd.

Author: dulek

tests/playbooks/roles/install-docker-registry/tasks/main.yml

@@ -33,6 +33,14 @@
     cmd: |
       cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
 
+- name: Create server certificate
+  shell:
+    executable: /bin/bash
+    chdir: "{{ ansible_user_dir }}/certs"
+    creates: "{{ ansible_user_dir }}/certs/server.pem"
+    cmd: |
+      cfssl gencert -config ca-config.json -profile server -ca ./ca.pem -ca-key ./ca-key.pem ca-csr.json | cfssljson -bare server
+
 - name: Run docker registry container
   shell:
     executable: /bin/bash
@@ -44,8 +52,8 @@
           --name registry \
           -v "{{ ansible_user_dir }}/certs":/certs \
           -e REGISTRY_HTTP_ADDR=0.0.0.0:443 \
-          -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/ca.pem \
-          -e REGISTRY_HTTP_TLS_KEY=/certs/ca-key.pem \
+          -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/server.pem \
+          -e REGISTRY_HTTP_TLS_KEY=/certs/server-key.pem \
           -p 443:443 \
           registry:2
       fi

tests/playbooks/roles/install-docker-registry/templates/ca-config.json.j2

@@ -9,15 +9,17 @@
         "usages": [
           "signing",
           "key encipherment",
-          "server auth"
+          "server auth",
+          "digital signature"
         ]
       },
       "client": {
         "expiry": "8760h",
         "usages": [
           "signing",
           "key encipherment",
-          "client auth"
+          "client auth",
+          "digital signature"
         ]
       }
     }

tests/playbooks/roles/install-k3s/tasks/main.yaml

@@ -88,19 +88,17 @@
         manage_etc_hosts: "localhost"
         package_update: true
         runcmd:
-          - curl -sSL https://get.docker.com/ | sh
+          - update-ca-certificates
           - mkdir -p /var/lib/rancher/k3s/agent/images/
           - curl -sSL https://github.com/k3s-io/k3s/releases/download/{{ k3s_release }}/k3s-airgap-images-amd64.tar -o /var/lib/rancher/k3s/agent/images/k3s-airgap-images.tar
           - curl -sSL https://github.com/k3s-io/k3s/releases/download/{{ k3s_release }}/k3s -o /usr/local/bin/k3s
           - curl -sSL https://get.k3s.io -o /var/lib/rancher/k3s/install.sh
           - chmod u+x /var/lib/rancher/k3s/install.sh /usr/local/bin/k3s
-          - INSTALL_K3S_SKIP_DOWNLOAD=true /var/lib/rancher/k3s/install.sh --docker --disable traefik --disable metrics-server --disable servicelb --disable-cloud-controller --kubelet-arg="cloud-provider=external" --tls-san {{ k3s_fip }} --token {{ cluster_token }}
+          - INSTALL_K3S_SKIP_DOWNLOAD=true /var/lib/rancher/k3s/install.sh --disable traefik --disable metrics-server --disable servicelb --disable-cloud-controller --kubelet-arg="cloud-provider=external" --tls-san {{ k3s_fip }} --token {{ cluster_token }}
         write_files:
-          - path: /etc/docker/daemon.json
+          - path: /usr/local/share/ca-certificates/registry-ca.crt
             content: |
-              {
-                "insecure-registries" : ["{{ ansible_default_ipv4.address }}"]
-              }
+      $(awk '{printf "        %s\n", $0}' < /root/certs/ca.pem)
       EOF
 
         # Create k3s master
@@ -126,19 +124,17 @@
         manage_etc_hosts: "localhost"
         package_update: true
         runcmd:
-          - curl -sSL https://get.docker.com/ | sh
+          - update-ca-certificates
           - mkdir -p /var/lib/rancher/k3s/agent/images/
           - curl -sSL https://github.com/k3s-io/k3s/releases/download/{{ release.stdout }}/k3s-airgap-images-amd64.tar -o /var/lib/rancher/k3s/agent/images/k3s-airgap-images.tar
           - curl -sSL https://github.com/k3s-io/k3s/releases/download/{{ release.stdout }}/k3s -o /usr/local/bin/k3s
           - curl -sSL https://get.k3s.io -o /var/lib/rancher/k3s/install.sh
           - chmod u+x /var/lib/rancher/k3s/install.sh /usr/local/bin/k3s
           - INSTALL_K3S_SKIP_DOWNLOAD=true K3S_URL=https://{{ k3s_fip }}:6443 K3S_TOKEN={{ cluster_token }} /var/lib/rancher/k3s/install.sh --docker --kubelet-arg="cloud-provider=external"
         write_files:
-          - path: /etc/docker/daemon.json
+          - path: /usr/local/share/ca-certificates/registry-ca.crt
             content: |
-              {
-                "insecure-registries" : ["{{ ansible_default_ipv4.address }}"]
-              }
+      $(awk '{printf "        %s\n", $0}' < /root/certs/ca.pem)
       EOF
 
         # Create k3s worker

tests/playbooks/test-csi-cinder-e2e.yaml

@@ -17,11 +17,11 @@
         - neutron
         - glance
         - cinder
-    - role: install-k3s
-      worker_node_count: 0
     - role: install-docker
     - role: install-docker-registry
       cert_hosts: ' ["{{ ansible_default_ipv4.address }}"]'
+    - role: install-k3s
+      worker_node_count: 0
     - role: install-cpo-occm
       run_e2e: false
       environment: "{{ global_env }}"

tests/playbooks/test-csi-manila-e2e.yaml

@@ -16,11 +16,11 @@
         - neutron
         - glance
         - manila
-    - role: install-k3s
-      worker_node_count: 0
     - role: install-docker
     - role: install-docker-registry
       cert_hosts: ' ["{{ ansible_default_ipv4.address }}"]'
+    - role: install-k3s
+      worker_node_count: 0
     - role: install-cpo-occm
       run_e2e: false
     - role: install-helm

tests/playbooks/test-occm-e2e.yaml

@@ -20,11 +20,11 @@
         - octavia
         - ovn-octavia
         - barbican
-    - role: install-k3s
-      worker_node_count: 0
     - role: install-docker
     - role: install-docker-registry
       cert_hosts: ' ["{{ ansible_default_ipv4.address }}"]'
+    - role: install-k3s
+      worker_node_count: 0
     - role: install-cpo-occm
       run_e2e: "{{ run_e2e }}"
       octavia_provider: "{{ octavia_provider }}"