Merge pull request #365 from ComputeCanada/incus

Add Incus to Magic Castle providers

Author: cmd-ntrf

README.md

@@ -21,6 +21,7 @@ From these new possibilities emerged an open-source software project named Magic
   - [Microsoft Azure](azure/README.md)
   - [Google Cloud Platform (GCP)](gcp/README.md)
   - [OVH Public Cloud (OVH)](ovh/README.md)
+  - [Incus](incus/README.md)
 - For more details, refer to [Magic Castle Documentation](docs)
 
 ## How Magic Castle Works

aws/infrastructure.tf

@@ -7,6 +7,7 @@ module "design" {
   source         = "../common/design"
   cluster_name   = var.cluster_name
   domain         = var.domain
+  image          = var.image
   instances      = var.instances
   min_disk_size  = 20
   pool           = var.pool
@@ -122,7 +123,7 @@ data "aws_ec2_instance_type" "instance_type" {
 resource "aws_instance" "instances" {
   for_each          = module.design.instances_to_build
   instance_type     = each.value.type
-  ami               = lookup(each.value, "image", var.image)
+  ami               = each.value.image
   user_data         = base64gzip(module.configuration.user_data[each.key])
   availability_zone = local.availability_zone
   placement_group   = contains(each.value.tags, "efa") ? aws_placement_group.efa_group.id : null

azure/infrastructure.tf

@@ -7,6 +7,7 @@ module "design" {
   source         = "../common/design"
   cluster_name   = var.cluster_name
   domain         = var.domain
+  image          = var.image
   instances      = var.instances
   min_disk_size  = 30
   pool           = var.pool
@@ -88,7 +89,7 @@ resource "azurerm_linux_virtual_machine" "instances" {
   }
 
   dynamic "source_image_reference" {
-    for_each = can(tomap(lookup(each.value, "image", var.image))) ? [lookup(each.value, "image", var.image)] : []
+    for_each = can(tomap(each.value.image)) ? [each.value.image] : []
     iterator = key
     content {
       publisher = key.value["publisher"]
@@ -97,7 +98,7 @@ resource "azurerm_linux_virtual_machine" "instances" {
       version   = lookup(key.value, "version", "latest")
     }
   }
-  source_image_id = can(tomap(lookup(each.value, "image", var.image))) ? null : tostring(lookup(each.value, "image", var.image))
+  source_image_id = can(tomap(each.value.image)) ? null : tostring(each.value.image)
 
   computer_name  = each.key
   admin_username = "azure"

common/configuration/main.tf

@@ -1,4 +1,11 @@
 variable "inventory" { }
+variable "post_inventory" {
+  default = {}
+}
+locals {
+  post_inventory = merge(var.inventory, var.post_inventory)
+}
+
 variable "config_git_url" { }
 variable "config_version" { }
 
@@ -49,14 +56,13 @@ locals {
   guest_passwd = var.guest_passwd != "" ? var.guest_passwd : try(random_pet.guest_passwd[0].id, "")
   public_keys = [for key in var.public_keys: trimspace(key)]
 
-  puppetservers = { for host, values in var.inventory: host => values.local_ip if contains(values.tags, "puppet")}
-  all_tags = toset(flatten([for key, values in var.inventory : values.tags]))
+  all_tags = toset(flatten([for key, values in local.post_inventory : values.tags]))
   tag_ip = { for tag in local.all_tags :
-    tag => [for key, values in var.inventory : values.local_ip if contains(values.tags, tag)]
+    tag => [for key, values in local.post_inventory : values.local_ip if contains(values.tags, tag)]
   }
 
   # add openssh public key to inventory
-  inventory = { for host, values in var.inventory:
+  final_inventory = { for host, values in local.post_inventory:
     host => merge(values, {
       hostkeys = {
         rsa     = chomp(tls_private_key.rsa[values.prefix].public_key_openssh)
@@ -67,10 +73,11 @@ locals {
 
   terraform_data  = yamlencode({
     terraform = {
-      instances = local.inventory
+      instances = local.final_inventory
       tag_ip    = local.tag_ip
       data      = {
         sudoer_username = var.sudoer_username
+        tf_public_key   = chomp(tls_private_key.ssh.public_key_openssh)
         public_keys     = local.public_keys
         cluster_name    = lower(var.cluster_name)
         domain_name     = var.domain_name
@@ -96,16 +103,11 @@ locals {
         domain_name           = var.domain_name
         puppetenv_git         = var.config_git_url,
         puppetenv_rev         = var.config_version,
-        puppetservers         = local.puppetservers,
+        puppetservers         = { for host, values in var.inventory: host => try(values.local_ip, "") if contains(values.tags, "puppet")}
         puppetserver_password = local.puppet_passwd,
         sudoer_username       = var.sudoer_username,
         ssh_authorized_keys   = local.public_keys
         tf_ssh_public_key     = tls_private_key.ssh.public_key_openssh
-        # If there is no bastion, the terraform data has to be packed with the user_data of the puppetserver.
-        # We do not packed it systematically because it increases the user-data size to a value that can be
-        # near or exceeds the cloud provider limit - AWS 16KB, Azure and OpenStack 64KB, GCP 256 KB.
-        include_tf_data       = ! contains(local.all_tags, var.bastion_tag)
-        terraform_data        = local.terraform_data
         terraform_facts       = local.terraform_facts
         skip_upgrade          = var.skip_upgrade
         puppetfile            = var.puppetfile
@@ -138,15 +140,15 @@ output "terraform_facts" {
 }
 
 output "puppetservers" {
-  value = local.puppetservers
+  value = { for host, values in local.final_inventory: host => values.local_ip if contains(values.tags, "puppet")}
 }
 
 output "guest_passwd" {
   value = local.guest_passwd
 }
 
 output "inventory" {
-  value = local.inventory
+  value = local.final_inventory
 }
 
 output "ssh_key" {
@@ -158,7 +160,7 @@ output "ssh_key" {
 
 output "bastions" {
   value = {
-    for host, values in var.inventory: host => values
+    for host, values in local.final_inventory: host => values
     if contains(values.tags, var.bastion_tag) && contains(values.tags, "public") &&  (!contains(values.tags, "pool"))
   }
 }

common/configuration/puppet.yaml

@@ -8,16 +8,21 @@ mounts:
 users:
   - name: tf
     system: true
+    passwd: '*'
     no_user_group: true
     homedir: /tmp
 %{ if contains(tags, "puppet") }
     sudo: "ALL = NOPASSWD: /usr/sbin/update_etc_puppetlabs.sh *.zip"
 %{ endif ~}
   - name: ${sudoer_username}
     groups: adm, wheel, systemd-journal
+    passwd: '*'
     homedir: /${sudoer_username}
-    selinux_user: unconfined_u
     sudo: ALL=(ALL) NOPASSWD:ALL
+%{ if cloud_provider != "incus" }
+    # most container images do not have selinux pre-installed
+    selinux_user: unconfined_u
+%{ endif }
     ssh_authorized_keys:
 %{ for key in ssh_authorized_keys ~}
       - ${key}
@@ -26,15 +31,23 @@ users:
 runcmd:
   - chmod 755 /etc # avoid issue with Rocky 9.4
   - test ! -d /${sudoer_username} && userdel -f -r ${sudoer_username} && cloud-init clean -r
+%{ if cloud_provider != "incus" }
+    # most container images do not have selinux pre-installed
   - restorecon -R /${sudoer_username}
+%{ endif }
+  - dnf -y install openssh openssh-server rsync
   - echo -e "Include /etc/ssh/sshd_config.d/50-authenticationmethods.conf" >> /etc/ssh/sshd_config
   - sed -i '/HostKey \/etc\/ssh\/ssh_host_ecdsa_key/ s/^#*/#/' /etc/ssh/sshd_config
   - chmod 644 /etc/ssh/ssh_host_*_key.pub
   - chgrp ssh_keys /etc/ssh/ssh_host_*_key.pub
   - systemctl restart sshd
 # Make sure puppet server can be reached by name early in the process if we need to debug.
+# For some provider, the ip address of the puppetserver is not known in advance, so we make
+# sure it is not empty before adding it to /etc/hosts
 %{ for host, ip in puppetservers ~}
+%{ if ip != "" ~}
   - echo "${ip} ${host}" >> /etc/hosts
+%{ endif ~}
 %{ endfor ~}
   # Enable fastest mirror for distribution using dnf package manager
   - dnf config-manager --setopt=fastestmirror=True --save
@@ -44,7 +57,7 @@ runcmd:
       # Install required packages in runcmd instead of packages to speedup configuration
       # of the admin user. This reduces the risk of Terraform timing out when trying to
       # upload the terraform_data.yaml
-      dnf -y install git pciutils unzip rsync
+      dnf -y install git pciutils unzip
       dnf -y remove cockpit\* firewalld --exclude=iptables
 %{ if ! skip_upgrade ~}
       # Upgrade all packages except Puppet if already installed
@@ -98,9 +111,6 @@ runcmd:
   - /opt/puppetlabs/puppet/bin/r10k puppetfile install --moduledir=/etc/puppetlabs/code/modules --puppetfile=/etc/puppetlabs/code/Puppetfile
 %{ endif ~}
 # Wait for Terraform to scp its YAML data
-%{ if ! include_tf_data ~}
-  - while [ ! -e "/etc/puppetlabs/data/terraform_data.yaml" ]; do echo "$(date -I'seconds') Waiting for terraform to scp terraform_data.yaml"; sleep 5; done
-%{ endif ~}
 %{ if node_name != keys(puppetservers)[0] }
   - sed -e '/certificate-authority-service/ s/^/#/' -i /etc/puppetlabs/puppetserver/services.d/ca.cfg
   - sed -e '/certificate-authority-disabled-service/ s/^#//' -i /etc/puppetlabs/puppetserver/services.d/ca.cfg
@@ -133,7 +143,8 @@ runcmd:
   - test -f /etc/magic-castle-release && systemctl start puppet || true
 
 write_files:
-  - content: restrict%{ if contains(tags, "puppet") },pty%{ else }%{ for host, ip in puppetservers },permitopen="${ip}:22"%{ endfor },port-forwarding,command="/sbin/nologin"%{ endif } ${tf_ssh_public_key}
+  # If the ip addresses of the puppet servers are not known in advance, we cannot restrict the ssh connection to them.
+  - content: restrict,%{ if contains(tags, "puppet") }pty%{ else }%{if length(compact(values(puppetservers))) == length(keys(puppetservers)) }%{ for host, ip in puppetservers }permitopen="${ip}:22"%{ endfor }%{ else }pty%{ endif },port-forwarding,command="/sbin/nologin"%{ endif } ${tf_ssh_public_key}
     path: /etc/ssh/authorized_keys.tf
     permissions: "0644"
   - content: |
@@ -183,7 +194,7 @@ write_files:
       if [ -f /usr/local/bin/consul ] && [ -f /usr/bin/jq ]; then
           /usr/local/bin/consul event -token=$(jq -r .acl.tokens.agent /etc/consul/config.json) -name=puppet $(date +%s)
       fi
-    permissions: 0755
+    permissions: "0755"
     path: /usr/sbin/update_etc_puppetlabs.sh
   - content: |
       ---
@@ -198,16 +209,6 @@ write_files:
     permissions: "0640"
   - path: /var/log/autosign.log
     permissions: "0640"
-%{ if include_tf_data ~}
-  - path: /etc/puppetlabs/data/terraform_data.yaml
-    content: |
-      ${indent(6, terraform_data)}
-    permissions: "0640"
-  - path: /etc/puppetlabs/facts/terraform_facts.yaml
-    content: |
-      ${indent(6, terraform_facts)}
-    permissions: "0640"
-%{ endif ~}
 %{ if puppetfile != "" ~}
   - path: /etc/puppetlabs/code/Puppetfile
     content: |

common/design/main.tf

@@ -14,6 +14,7 @@ locals {
       for prefix, attrs in var.instances : [
         for i in range(lookup(attrs, "count", 1)) : {
           (format("%s%d", prefix, i + 1)) = merge(
+            { image = var.image },
             { disk_size = max(var.min_disk_size, [for tag in attrs.tags: lookup(local.min_disk_size_per_tags, tag, 0)]...)},
             { for attr, value in attrs : attr => value if ! contains(["count"], attr) },
             {

common/design/variables.tf

@@ -4,4 +4,5 @@ variable "instances" { }
 variable "volumes" { }
 variable "pool" { }
 variable "firewall_rules" { }
-variable "min_disk_size" { }
+variable "min_disk_size" { }
+variable "image" { }

docs/README.md

@@ -455,6 +455,17 @@ Here is an example:
 The image name can be a regular expression. If more than one image is returned by the query
 to OpenStack, the most recent is selected.
 
+#### 4.6.4 Incus
+
+The source of the images is [images.linuxcontainers.org/](https://images.linuxcontainers.org/).
+Make sure to select the `cloud` variant of the distribution image since only these images
+have cloud-init installed and enabled, which is required for bootstrapping Magic Castle.
+
+Since the cloud variant of images is a requirement for Magic Castle, it is implemented
+that an Incus image name that does not end with `/cloud` has to be a local image
+created using the process documented in section
+[10.12 - Create a compute node image](#1012-create-a-compute-node-image).
+
 ### 4.7 instances
 
 The `instances` variable is a map that defines the virtual machines that will form
@@ -1095,6 +1106,21 @@ find it automatically. Can be used to force a v4 subnet when both v4 and v6 exis
 
 **Post build modification effect**: rebuild of all instances at next `terraform apply`.
 
+### 5.5 Incus
+
+#### forward_proxy (optional)
+
+**default value**: false
+
+When enabled, the network connections for ports with the `proxy` tags in firewall rules
+are forwarded between the incus host and the proxy instance. To do so, an
+[incus proxy device](https://linuxcontainers.org/incus/docs/main/reference/devices_proxy/)
+is added to the instance for each port.
+
+This setting can be enabled on at most one cluster per incus host.
+
+**Post build modification effect**: add or remove devices forwarding proxy ports.
+
 ## 6. DNS Configuration
 
 Some functionalities in Magic Castle require the registration of DNS records under the
@@ -1659,6 +1685,11 @@ The script `prepare4image.sh` executes the following steps in order:
   12. Remove cloud-init's logs and artifacts so it can re-run
   13. Power off the machine
 
+**Warning**: When using Incus, running `prepare4image.sh` on instance is an irreversible process.
+The script removes `/var/lib/cloud` which is only created when the instance is created. Therefore,
+to make the instance work properly again, you will either have to taint it or change its image
+attribute to the one you have just created.
+
 #### 10.12.2 Create the image
 
 Once the instance is powered off, access your cloud provider dashboard, find the instance
@@ -1667,6 +1698,7 @@ and follow the provider's instructions to create the image.
 - [AWS](https://docs.aws.amazon.com/toolkit-for-visual-studio/latest/user-guide/tkv-create-ami-from-instance.html)
 - [Azure](https://learn.microsoft.com/en-us/azure/virtual-machines/capture-image-portal)
 - [GCP](https://cloud.google.com/compute/docs/machine-images/create-machine-images#create-image-from-instance)
+- [Incus](https://linuxcontainers.org/incus/docs/main/howto/images_create/)
 - [OpenStack](https://docs.openstack.org/horizon/latest/user/launch-instances.html#create-an-instance-snapshot)
 - [OVH](https://blog.ovhcloud.com/create-and-use-openstack-snapshots/)
 

examples/incus/data.yaml

@@ -0,0 +1,20 @@
+---
+profile::nfs::server::export_paths: ['/home', '/project', '/scratch']
+profile::nfs::client::share_names: ['home', 'project', 'scratch']
+profile::slurm::controller::selinux_context: ""
+
+magic_castle::site::all:
+  - profile::base
+  - profile::consul
+  - profile::freeipa
+  - profile::users::local
+  - profile::sssd::client
+  - profile::metrics::node_exporter
+  - profile::rsyslog::client
+  - profile::volumes
+# - swap_file
+# We remove swap when running in containers
+# because when comes time to destroy the cluster
+# incus is unable to run swap_off on the instance
+# swap file (it tries a simple rm instead)
+# and it results in a crash of the terraform destroy.