fix: adjust Flatcar aks-sysext to not use /etc or /opt/bin

Hosting a directory-based sysext from /etc is a problem because Flatcar
will use confext soon, which will make /etc an overlayfs. It is now
hosted from /mnt/aks-sysext instead.

Pointing the /usr/local/bin symlink at /opt/bin is also confusing
because AKS now installs several binaries here. It just needs to point
somewhere writeable, so this now points under /mnt/aks-sysext-rw.

The symlink now covers the whole of /usr/local, not just /usr/local/bin,
as there may be components that need to write to other directories.
Nothing in Agent Baker should write to /usr/local now, but VM extensions
may still do this.

Signed-off-by: James Le Cuirot <jlecuirot@microsoft.com>

Author: chewi

vhdbuilder/packer/flatcar-customdata.json

@@ -1 +1 @@
-{"ignition":{"version":"3.4.0"},"kernelArguments":{"shouldNotExist":["flatcar.autologin"]},"storage":{"files":[{"path":"/etc/extensions/aks-sysext/usr/lib/extension-release.d/extension-release.aks-sysext","contents":{"compression":"","source":"data:,ID%3Dflatcar%0ASYSEXT_LEVEL%3D1.0%0A"},"mode":420},{"overwrite":true,"path":"/etc/flatcar/update.conf","contents":{"compression":"","source":"data:,SERVER%3Ddisabled%0A"},"mode":420},{"path":"/etc/systemd/system/containerd.service.d/50-default-config.conf","contents":{"compression":"","source":"data:,%5BService%5D%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0A"},"mode":420},{"path":"/etc/tmpfiles.d/protocols.conf","contents":{"compression":"","source":"data:,C%20%2Fetc%2Fprotocols%20-%20-%20-%20-%20%2Fusr%2Fshare%2Fbaselayout%2Fprotocols%0A"},"mode":420},{"overwrite":true,"path":"/etc/nsswitch.conf","contents":{"compression":"gzip","source":"data:;base64,H4sIAAAAAAAC/3yP0U7DMAxF3/0VkXhn7/sbk3o0ookjX4dof4+yFJgGXZ+qe46i45dwEo+nAvTkcX2NWi5nospAX87h9l3SJggNNn8ABFzhkhfCyov2Q4/eTVvd8bN3aFU4DsR8HbRwlrAUUBHvah/TflAHJ4h9pij/CVRNXaNuN/jArMbvgj+MxFcx3F86QjLjPmRMvyfP6U3VKxvn4c2Jm2vWVvwngnhLjD15n74CAAD//8ytimubAQAA"},"mode":420}],"links":[{"path":"/etc/extensions/aks-sysext/usr/local/bin","hard":false,"target":"/opt/bin"},{"path":"/etc/containerd/config.toml","hard":false,"target":"/usr/share/containerd/config.toml"}]},"systemd":{"units":[{"dropins":[{"contents":"[Unit]\nConditionPathIsSymbolicLink=\nConditionPathIsSymbolicLink=/etc/ssl/certs/ca-certificates.crt\n","name":"10-ensure-ca-file.conf"}],"enabled":true,"name":"update-ca-certificates.service"}]}}
+{"ignition":{"version":"3.4.0"},"kernelArguments":{"shouldNotExist":["flatcar.autologin"]},"storage":{"directories":[{"path":"/mnt/aks-sysext-rw/usr/local"},{"path":"/mnt/aks-sysext-rw/usr/local/bin"},{"path":"/mnt/aks-sysext-rw/usr/local/lib"},{"path":"/mnt/aks-sysext-rw/usr/local/lib64"},{"path":"/mnt/aks-sysext-rw/usr/local/sbin"},{"path":"/mnt/aks-sysext-rw/usr/local/share"}],"files":[{"path":"/mnt/aks-sysext/usr/lib/extension-release.d/extension-release.aks-sysext","contents":{"compression":"","source":"data:,ID%3Dflatcar%0ASYSEXT_LEVEL%3D1.0%0A"}},{"overwrite":true,"path":"/etc/flatcar/update.conf","contents":{"compression":"","source":"data:,SERVER%3Ddisabled%0A"}},{"path":"/etc/systemd/system/containerd.service.d/50-default-config.conf","contents":{"compression":"","source":"data:,%5BService%5D%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0A"}},{"path":"/etc/tmpfiles.d/protocols.conf","contents":{"compression":"","source":"data:,C%20%2Fetc%2Fprotocols%20-%20-%20-%20-%20%2Fusr%2Fshare%2Fbaselayout%2Fprotocols%0A"}},{"overwrite":true,"path":"/etc/nsswitch.conf","contents":{"compression":"gzip","source":"data:;base64,H4sIAAAAAAAC/3yP0U7DMAxF3/0VkXhn7/sbk3o0ookjX4dof4+yFJgGXZ+qe46i45dwEo+nAvTkcX2NWi5nospAX87h9l3SJggNNn8ABFzhkhfCyov2Q4/eTVvd8bN3aFU4DsR8HbRwlrAUUBHvah/TflAHJ4h9pij/CVRNXaNuN/jArMbvgj+MxFcx3F86QjLjPmRMvyfP6U3VKxvn4c2Jm2vWVvwngnhLjD15n74CAAD//8ytimubAQAA"}}],"links":[{"path":"/etc/extensions/aks-sysext","hard":false,"target":"/mnt/aks-sysext"},{"path":"/mnt/aks-sysext/usr/local","hard":false,"target":"/mnt/aks-sysext-rw/usr/local"},{"path":"/etc/containerd/config.toml","hard":false,"target":"/usr/share/containerd/config.toml"}]},"systemd":{"units":[{"dropins":[{"contents":"[Unit]\nConditionPathIsSymbolicLink=\nConditionPathIsSymbolicLink=/etc/ssl/certs/ca-certificates.crt\n","name":"10-ensure-ca-file.conf"}],"enabled":true,"name":"update-ca-certificates.service"}]}}

vhdbuilder/packer/flatcar-customdata.yaml

@@ -4,34 +4,36 @@ kernel_arguments:
   should_not_exist:
     - flatcar.autologin
 storage:
+  directories:
+  - path: /mnt/aks-sysext-rw/usr/local
+  - path: /mnt/aks-sysext-rw/usr/local/bin
+  - path: /mnt/aks-sysext-rw/usr/local/lib
+  - path: /mnt/aks-sysext-rw/usr/local/lib64
+  - path: /mnt/aks-sysext-rw/usr/local/sbin
+  - path: /mnt/aks-sysext-rw/usr/local/share
   files:
-  - path: /etc/extensions/aks-sysext/usr/lib/extension-release.d/extension-release.aks-sysext
-    mode: 0644
+  - path: /mnt/aks-sysext/usr/lib/extension-release.d/extension-release.aks-sysext
     contents:
       inline: |
         ID=flatcar
         SYSEXT_LEVEL=1.0
   - path: /etc/flatcar/update.conf
-    mode: 0644
     overwrite: true
     contents:
       inline: |
         SERVER=disabled
   - path: /etc/systemd/system/containerd.service.d/50-default-config.conf
-    mode: 0644
     contents:
       inline: |
         [Service]
         Environment=CONTAINERD_CONFIG=/etc/containerd/config.toml
   # Temporary: for compatibility with azure-npm and before https://github.com/flatcar/Flatcar/issues/1852 is resolved
   # simply copy the protocols file from baselayout to /etc.
   - path: /etc/tmpfiles.d/protocols.conf
-    mode: 0644
     contents:
       inline: |
         C /etc/protocols - - - - /usr/share/baselayout/protocols
   - path: /etc/nsswitch.conf
-    mode: 0644
     overwrite: true
     contents:
       inline: |
@@ -55,8 +57,11 @@ storage:
         automount:   files
         aliases:     files
   links:
-  - path: /etc/extensions/aks-sysext/usr/local/bin
-    target: /opt/bin
+  - path: /etc/extensions/aks-sysext
+    target: /mnt/aks-sysext
+    hard: false
+  - path: /mnt/aks-sysext/usr/local
+    target: /mnt/aks-sysext-rw/usr/local
     hard: false
   - path: /etc/containerd/config.toml
     target: /usr/share/containerd/config.toml