feat: node-exporter into vhd build (#7704)

Author: chmill-zz

.github/renovate.json

@@ -215,6 +215,17 @@
         "nilo19"
       ]
     },
+    {
+      "matchPackageNames": [
+        "node-exporter-kubernetes"
+      ],
+      "matchUpdateTypes": [
+        "patch"
+      ],
+      "automerge": false,
+      "enabled": true,
+      "groupName": "node-exporter-kubernetes"
+    },
     {
       "matchPackageNames": [
         "containernetworking/azure-cni"

e2e/validation.go

@@ -46,6 +46,7 @@ func ValidateCommonLinux(ctx context.Context, s *Scenario) {
 	ValidateRxBufferDefault(ctx, s)
 	ValidateKernelLogs(ctx, s)
 	ValidateScriptlessCSECmd(ctx, s)
+	ValidateNodeExporter(ctx, s)
 
 	ValidateSysctlConfig(ctx, s, map[string]string{
 		"net.ipv4.tcp_retries2":             "8",

e2e/validators.go

@@ -1405,6 +1405,67 @@ func ValidateNodeProblemDetector(ctx context.Context, s *Scenario) {
 	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "Node Problem Detector (NPD) service validation failed")
 }
 
+func ValidateNodeExporter(ctx context.Context, s *Scenario) {
+	s.T.Helper()
+
+	skipFile := "/etc/node-exporter.d/skip_vhd_node_exporter"
+	serviceName := "node-exporter.service"
+
+	// Check if node-exporter is installed on this VHD by looking for the skip sentinel file.
+	// The skip file is only present on VHDs that have node-exporter installed (Ubuntu, Mariner, Azure Linux).
+	// Flatcar, OSGuard, and older VHDs do not have node-exporter installed and will not have the skip file.
+	if !fileExist(ctx, s, skipFile) {
+		s.T.Logf("Skipping node-exporter validation: sentinel file %s not found (VHD does not have node-exporter installed)", skipFile)
+		return
+	}
+
+	s.T.Logf("skip_vhd_node_exporter sentinel file found, validating node-exporter installation")
+
+	// Validate service is running
+	ValidateSystemdUnitIsRunning(ctx, s, serviceName)
+	ValidateSystemdUnitIsNotFailed(ctx, s, serviceName)
+
+	// Validate service is enabled
+	execScriptOnVMForScenarioValidateExitCode(ctx, s, fmt.Sprintf("systemctl is-enabled %s", serviceName), 0, fmt.Sprintf("%s should be enabled", serviceName))
+
+	// Validate binary exists and is executable
+	// The binary is installed at /usr/bin and symlinked to /opt/bin for consistency with other binaries (kubelet, etc.)
+	ValidateFileExists(ctx, s, "/usr/bin/node-exporter")
+	ValidateFileExists(ctx, s, "/opt/bin/node-exporter")
+	ValidateFileExists(ctx, s, "/opt/bin/node-exporter-startup.sh")
+
+	// Validate configuration files exist
+	ValidateFileExists(ctx, s, skipFile)
+	ValidateFileExists(ctx, s, "/etc/node-exporter.d/web-config.yml")
+
+	// Validate that node-exporter is listening on port 19100
+	// We verify the port is open using ss/netstat rather than making a full mTLS request,
+	// since the e2e test environment may not have the correct client certs set up.
+	// The mTLS configuration is validated by checking that the web-config.yml exists
+	// and contains the expected TLS settings.
+	s.T.Logf("Validating node-exporter is listening on port 19100")
+	command := []string{
+		"set -ex",
+		"NODE_IP=$(hostname -I | awk '{print $1}')",
+		// Verify node-exporter is listening on port 19100
+		"ss -tlnp | grep -q ':19100' || netstat -tlnp | grep -q ':19100'",
+	}
+	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "node-exporter should be listening on port 19100")
+
+	// Verify the web-config.yml has proper TLS configuration
+	s.T.Logf("Validating node-exporter TLS configuration")
+	tlsCommand := []string{
+		"set -ex",
+		// Verify web-config.yml contains TLS settings
+		"grep -q 'tls_server_config' /etc/node-exporter.d/web-config.yml",
+		"grep -q 'client_auth_type' /etc/node-exporter.d/web-config.yml",
+		"grep -q 'client_ca_file' /etc/node-exporter.d/web-config.yml",
+	}
+	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(tlsCommand, "\n"), 0, "node-exporter TLS config should be properly configured")
+
+	s.T.Logf("node-exporter validation passed")
+}
+
 func ValidateNPDFilesystemCorruption(ctx context.Context, s *Scenario) {
 	command := []string{
 		"set -ex",

parts/common/components.json

@@ -2045,6 +2045,61 @@
           }
         }
       }
+    },
+    {
+      "name": "node-exporter",
+      "downloadLocation": "/opt/node-exporter",
+      "downloadURIs": {
+        "ubuntu": {
+          "r2404": {
+            "versionsV2": [
+              {
+                "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=24.04",
+                "latestVersion": "1.9.1-ubuntu24.04u12"
+              }
+            ]
+          },
+          "r2204": {
+            "versionsV2": [
+              {
+                "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=22.04",
+                "latestVersion": "1.9.1-ubuntu22.04u12"
+              }
+            ]
+          },
+          "r2004": {
+            "versionsV2": [
+              {
+                "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=20.04",
+                "latestVersion": "1.9.1-ubuntu20.04u12"
+              }
+            ]
+          }
+        },
+        "azurelinux": {
+          "v3.0": {
+            "versionsV2": [
+              {
+                "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=node-exporter-kubernetes, os=azurelinux, release=3.0",
+                "latestVersion": "1.9.1-12.azl3"
+              }
+            ]
+          },
+          "OSGUARD/v3.0": {
+            "versionsV2": []
+          }
+        },
+        "mariner": {
+          "current": {
+            "versionsV2": []
+          }
+        },
+        "flatcar": {
+          "current": {
+            "versionsV2": []
+          }
+        }
+      }
     }
   ],
   "OCIArtifacts": [

parts/linux/cloud-init/artifacts/cse_config.sh

@@ -807,6 +807,25 @@ EOF
     systemctlEnableAndStart mig-partition 300
 }
 
+configureNodeExporter() {
+    echo "Configuring Node Exporter"
+    # Check for skip file to determine if node-exporter was installed on this VHD
+    if [ ! -f /etc/node-exporter.d/skip_vhd_node_exporter ]; then
+        echo "Node Exporter assets not found on this VHD (missing /etc/node-exporter.d/skip_vhd_node_exporter); skipping configuration."
+        return 0
+    fi
+
+    if ! systemctlEnableAndStart node-exporter 30; then
+        echo "Failed to start node-exporter service"
+        return $ERR_NODE_EXPORTER_START_FAIL
+    fi
+    if ! systemctlEnableAndStart node-exporter-restart.path 30; then
+        echo "Failed to start node-exporter-restart.path"
+        return $ERR_NODE_EXPORTER_START_FAIL
+    fi
+    echo "Node Exporter started successfully"
+}
+
 ensureSysctl() {
     SYSCTL_CONFIG_FILE=/etc/sysctl.d/999-sysctl-aks.conf
     mkdir -p "$(dirname "${SYSCTL_CONFIG_FILE}")"

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -78,6 +78,8 @@ ERR_ENABLE_MANAGED_GPU_EXPERIENCE=123 # Error confguring managed GPU experience
 # Error code 124 is returned when a `timeout` command times out, and --preserve-status is not specified: https://man7.org/linux/man-pages/man1/timeout.1.html
 ERR_VHD_BUILD_ERROR=125 # Reserved for VHD CI exit conditions
 
+ERR_NODE_EXPORTER_START_FAIL=128 # Error starting or enabling node-exporter service
+
 ERR_SWAP_CREATE_FAIL=130 # Error allocating swap file
 ERR_SWAP_CREATE_INSUFFICIENT_DISK_SPACE=131 # Error insufficient disk space for swap file creation
 

parts/linux/cloud-init/artifacts/cse_main.sh

@@ -473,6 +473,8 @@ function nodePrep {
 
     logs_to_events "AKS.CSE.ensureKubelet" ensureKubelet
 
+    logs_to_events "AKS.CSE.configureNodeExporter" configureNodeExporter
+
     if $REBOOTREQUIRED; then
         echo 'reboot required, rebooting node in 1 minute'
         /bin/bash -c "shutdown -r 1 &"

parts/linux/cloud-init/artifacts/node-exporter/baseline/etc/node-exporter.d/web-config.yml

@@ -0,0 +1,5 @@
+tls_server_config:
+  cert_file: "/etc/kubernetes/certs/kubeletserver.crt"
+  key_file: "/etc/kubernetes/certs/kubeletserver.key"
+  client_auth_type: "RequireAndVerifyClientCert"
+  client_ca_file: "/etc/kubernetes/certs/ca.crt"

parts/linux/cloud-init/artifacts/node-exporter/baseline/etc/systemd/system/node-exporter-restart.path

@@ -0,0 +1,9 @@
+[Path]
+# Watch server cert paths - one will exist depending on whether kubelet serving cert rotation is enabled
+# Rotation enabled: kubelet-server-current.pem (symlink updated on rotation)
+# Rotation disabled: kubeletserver.crt (static cert)
+PathModified=/var/lib/kubelet/pki/kubelet-server-current.pem
+PathModified=/etc/kubernetes/certs/kubeletserver.crt
+
+[Install]
+WantedBy=multi-user.target

parts/linux/cloud-init/artifacts/node-exporter/baseline/etc/systemd/system/node-exporter-restart.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Restart node-exporter when certificates change
+
+[Service]
+Type=oneshot
+ExecStart=/bin/systemctl restart node-exporter.service