Fix race condition causing sshd start failure during provisioning (#460)

* Fix race condition causing sshd start failure during provisioning

 * Run first-boot tasks via systemd so sshd never races with host-key
   regeneration. The old `rc.local` script ran after network.target, but
   in parallel with other regular system services, like ssh.service.
   Therefore, ssh.service often started (and restarted) while
   `/root/firstboot.sh` was deleting keys. cloud-init’s set-passwords
   module made this worse by restarting ssh mid-run.
 * Replace `rc.local` with a oneshot firstboot.service (delete keys,
   create new keys, reconfigure sysstat) that runs Before=ssh.service
   and leaves the `/root/firstboot_done` file as a marker.
 * Add a cloud-config.service drop-in so cloud-init's config stage waits
   for firstboot.service, and
 * Update walinuxagent.service to wait for firstboot.service, ensuring
   ssh keys have been regenerated. This guarantees sshd, cloud-init, and
   WALinuxAgent all start only after the first-boot tasks succeed.

* Invoke firstboot script in firstboot.service

 * Move multiple firstboot commands back into a firstboot script in case
   we need to extend this in the future.
 * Use systemd's `ConditionFirstBoot` native option to ensure the script
   is only run on firstboot, which allows us to get rid of the
   file-based approach.

Author: s4heid

stemcell_builder/stages/base_ssh/assets/10-ssh-firstboot-done.conf

@@ -1,2 +1,3 @@
 [Unit]
-ConditionPathExists=/root/firstboot_done
+Wants=firstboot.service
+After=firstboot.service

stemcell_builder/stages/base_ubuntu_firstboot/apply.sh

@@ -5,7 +5,12 @@ set -e
 base_dir=$(readlink -nf $(dirname $0)/../..)
 source $base_dir/lib/prelude_apply.bash
 
-cp $assets_dir/etc/rc.local $chroot/etc/rc.local
-cp $assets_dir/root/firstboot.sh $chroot/root/firstboot.sh
-chmod u+x "${chroot}/etc/rc.local"
-chmod 0755 $chroot/root/firstboot.sh
+install -D -m 0755 \
+  $assets_dir/root/firstboot.sh \
+  $chroot/root/firstboot.sh
+
+install -D -m 0644 \
+  $assets_dir/etc/systemd/system/firstboot.service \
+  $chroot/etc/systemd/system/firstboot.service
+
+run_in_chroot $chroot "systemctl enable firstboot.service"

stemcell_builder/stages/base_ubuntu_firstboot/assets/etc/rc.local

@@ -0,0 +1,12 @@
+[Unit]
+Description=Run first boot tasks
+ConditionFirstBoot=yes
+Before=ssh.service
+
+[Service]
+Type=oneshot
+ExecStart=/root/firstboot.sh
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target

stemcell_builder/stages/base_ubuntu_firstboot/assets/etc/systemd/system/firstboot.service

@@ -1,6 +1,7 @@
 #!/bin/sh
+set -e
 
 rm -f /etc/ssh/ssh_host*key*
+ssh-keygen -A -v
 
-dpkg-reconfigure -fnoninteractive -pcritical openssh-server
 dpkg-reconfigure -fnoninteractive sysstat

stemcell_builder/stages/base_ubuntu_firstboot/assets/root/firstboot.sh

@@ -46,12 +46,16 @@ cat > $chroot/etc/logrotate.d/waagent <<EOS
 }
 EOS
 
-#setup cloud-init
+# Setup cloud-init
 rm $chroot/etc/cloud/*.cfg
 rm $chroot/etc/cloud/cloud.cfg.d/*.cfg
 cp -f $dir/assets/etc/cloud-init/cloud.cfg $chroot/etc/cloud/cloud.cfg
 cp -f $dir/assets/etc/cloud-init/*-*.cfg $chroot/etc/cloud/cloud.cfg.d/
 
+# Ensures that cloud-init waits until host keys have been regenerated.
+mkdir -p $chroot/etc/systemd/system/cloud-config.service.d
+cp -f $dir/assets/etc/systemd/system/cloud-config.service.d/firstboot-blocker.conf \
+      $chroot/etc/systemd/system/cloud-config.service.d/firstboot-blocker.conf
 
 # this will append the following two relevant lines (plus a few commented out lines)
 # to the default-conf:

stemcell_builder/stages/system_azure_init/apply.sh

@@ -0,0 +1,3 @@
+[Unit]
+Wants=firstboot.service
+After=firstboot.service

stemcell_builder/stages/system_azure_init/assets/etc/systemd/system/cloud-config.service.d/firstboot-blocker.conf

@@ -7,17 +7,15 @@
 [Unit]
 Description=Azure Linux Agent
 
-After=network-online.target
-Wants=network-online.target ssh.service sshd-keygen.service
+# NON-DEFAULT: Must run after the firstboot.service, which regenerates the ssh keys
+After=firstboot.service network-online.target cloud-init.service
+Wants=network-online.target sshd.service sshd-keygen.service
 
 ConditionFileIsExecutable=/usr/sbin/waagent
 ConditionPathExists=/etc/waagent.conf
 
 [Service]
 Type=simple
-# stemcells on Azure re-generate the SSH Hostkey upon first reboot
-# waagent has to wait until the file was recreated
-ExecStartPre=/bin/bash -c "while [ ! -f /root/firstboot_done ]; do sleep 1; done"
 ExecStart=/usr/bin/python3 -u /usr/sbin/waagent -daemon
 Restart=always
 Slice=azure.slice