feat: verify binary checksums before execution (#797)

Signed-off-by: Rado M <radkomih@gmail.com>

Author: radkomih

internal/state/state_reader.go

@@ -87,6 +87,21 @@ type PromptDefaultsDoc struct {
 	} `yaml:"state"`
 }
 
+// SoftwareVersionsDoc is the minimal YAML shape for reading each installed host
+// component's recorded version from machineState.software. The map key (installer
+// key, e.g. "crio") and the nested name (catalog artifact name, e.g. "cri-o") may
+// differ, so softwareVersionFromDoc matches on either.
+type SoftwareVersionsDoc struct {
+	State struct {
+		MachineState struct {
+			Software map[string]struct {
+				Name    string `yaml:"name"`
+				Version string `yaml:"version"`
+			} `yaml:"software"`
+		} `yaml:"machineState"`
+	} `yaml:"state"`
+}
+
 // ReadProvisionerVersionFromDisk extracts the provisioner version from the on-disk state file
 // without loading the full state into memory. Returns an empty string when no state file exists.
 func ReadProvisionerVersionFromDisk() (string, error) {
@@ -103,6 +118,40 @@ func ReadProvisionerVersionFromDisk() (string, error) {
 	return doc.State.Provisioner.Version, nil
 }
 
+// ReadSoftwareVersionFromDisk reads a single host component's recorded version
+// (by catalog artifact name) from the on-disk state file without loading the
+// full state. Returns an empty string when the state file or component is absent.
+func ReadSoftwareVersionFromDisk(name string) (string, error) {
+	data, err := readStateFileBytes()
+	if err != nil || data == nil {
+		return "", err
+	}
+
+	var doc SoftwareVersionsDoc
+	if err := unmarshalStateDoc(data, &doc); err != nil {
+		return "", err
+	}
+
+	return softwareVersionFromDoc(doc, name), nil
+}
+
+// softwareVersionFromDoc resolves a component's recorded version, preferring a
+// direct map-key hit and falling back to matching an entry's recorded name (so a
+// lookup by artifact name finds an entry keyed by its installer key). Returns an
+// empty string when the component is not recorded.
+func softwareVersionFromDoc(doc SoftwareVersionsDoc, name string) string {
+	software := doc.State.MachineState.Software
+	if entry, ok := software[name]; ok {
+		return entry.Version
+	}
+	for _, entry := range software {
+		if entry.Name == name {
+			return entry.Version
+		}
+	}
+	return ""
+}
+
 // BlockNodeSummary holds the prompt-relevant fields from BlockNodeState
 // read from the on-disk state file.  HelmReleaseInfo is yaml:",inline"
 // inside BlockNodeState, so its keys (name, namespace, version) live

internal/state/state_reader_test.go

@@ -25,6 +25,42 @@ state:
 	}
 }
 
+func TestReadSoftwareVersion_ParsesRecordedVersion(t *testing.T) {
+	// cri-o is stored under the installer key "crio", not its artifact name.
+	data := []byte(`
+state:
+  machineState:
+    software:
+      cilium:
+        name: cilium
+        version: "0.18.7"
+        installed: true
+      crio:
+        name: cri-o
+        version: "1.30.0"
+`)
+	var doc SoftwareVersionsDoc
+	if err := unmarshalStateDoc(data, &doc); err != nil {
+		t.Fatalf("unexpected parse error: %v", err)
+	}
+	// Direct map-key hit.
+	if got := softwareVersionFromDoc(doc, "cilium"); got != "0.18.7" {
+		t.Fatalf("expected cilium version '0.18.7', got %q", got)
+	}
+	// Artifact name resolves via the name fallback despite the differing map key.
+	if got := softwareVersionFromDoc(doc, "cri-o"); got != "1.30.0" {
+		t.Fatalf("expected cri-o version '1.30.0' via name fallback, got %q", got)
+	}
+	// The installer key still resolves directly too.
+	if got := softwareVersionFromDoc(doc, "crio"); got != "1.30.0" {
+		t.Fatalf("expected cri-o version '1.30.0' via map key, got %q", got)
+	}
+	// Unrecorded component resolves to the empty string.
+	if got := softwareVersionFromDoc(doc, "teleport"); got != "" {
+		t.Fatalf("expected empty version for unrecorded component, got %q", got)
+	}
+}
+
 func TestReadPromptDefaults_ParsesProfile(t *testing.T) {
 	data := []byte(`
 state:

internal/workflows/cluster.go

@@ -42,6 +42,7 @@ func kubernetesSetupWorkflow(mr software.MachineRuntime) *automa.WorkflowBuilder
 
 		// kubelet
 		steps.SetupKubelet(mr),
+		steps.VerifyExecutablesStep("kubelet"),
 		steps.SetupSystemdService(software.KubeletServiceName),
 
 		// setup cli tools
@@ -51,6 +52,7 @@ func kubernetesSetupWorkflow(mr software.MachineRuntime) *automa.WorkflowBuilder
 
 		// CRI-O
 		steps.SetupCrio(mr),
+		steps.VerifyExecutablesStep(software.CrioArtifactName),
 		steps.SetupSystemdService(software.CrioServiceName),
 
 		// kubeadm

internal/workflows/migration_cilium_acceleration.go

@@ -55,6 +55,7 @@ var (
 	reconfigureCiliumConfig = software.ReconfigureCiliumConfig
 	kubernetesInstalled     = defaultKubernetesInstalled
 	readCiliumState         = defaultReadCiliumState
+	verifyCiliumExecutable  = func() error { return software.VerifyExecutables("cilium") }
 )
 
 // CiliumAccelerationMigration re-renders cilium-config.yaml and runs `cilium upgrade`
@@ -117,6 +118,11 @@ func (m *CiliumAccelerationMigration) Execute(ctx context.Context, mctx *migrati
 		Str("to", disabledAcceleration).
 		Msg("Applying cilium-config (loadBalancer.acceleration=disabled) to existing cluster via 'cilium upgrade'")
 
+	// Verify the cilium binary before executing it.
+	if err := verifyCiliumExecutable(); err != nil {
+		return err
+	}
+
 	upgrade := []string{
 		fmt.Sprintf("/usr/bin/sudo %s/cilium upgrade --values %s --wait",
 			models.Paths().SandboxBinDir, configPath),

internal/workflows/migration_cilium_acceleration_test.go

@@ -51,9 +51,9 @@ func TestCiliumAccelerationMigration_Applies(t *testing.T) {
 }
 
 func TestCiliumAccelerationMigration_Execute(t *testing.T) {
-	origK8s, origState, origReconfigure, origShell := kubernetesInstalled, readCiliumState, reconfigureCiliumConfig, runShell
+	origK8s, origState, origReconfigure, origShell, origVerify := kubernetesInstalled, readCiliumState, reconfigureCiliumConfig, runShell, verifyCiliumExecutable
 	t.Cleanup(func() {
-		kubernetesInstalled, readCiliumState, reconfigureCiliumConfig, runShell = origK8s, origState, origReconfigure, origShell
+		kubernetesInstalled, readCiliumState, reconfigureCiliumConfig, runShell, verifyCiliumExecutable = origK8s, origState, origReconfigure, origShell, origVerify
 	})
 
 	m := NewCiliumAccelerationMigration()
@@ -84,8 +84,9 @@ func TestCiliumAccelerationMigration_Execute(t *testing.T) {
 		assert.False(t, ran, "must not run before Kubernetes is installed")
 	})
 
-	// Remaining cases assume Kubernetes is present.
+	// Remaining cases assume Kubernetes is present and cilium verification passes.
 	kubernetesInstalled = func() bool { return true }
+	verifyCiliumExecutable = func() error { return nil }
 
 	t.Run("skips when Cilium is not installed", func(t *testing.T) {
 		var ran bool
@@ -128,4 +129,19 @@ func TestCiliumAccelerationMigration_Execute(t *testing.T) {
 		assert.Contains(t, cmd, "--values")
 		assert.Contains(t, cmd, "--wait")
 	})
+
+	t.Run("aborts before upgrade when the cilium binary fails verification", func(t *testing.T) {
+		var ran bool
+		var cmd string
+		stateReturns(true, "best-effort", nil)
+		reconfigureCiliumConfig = func() (string, error) {
+			return "/opt/solo/weaver/sandbox/etc/weaver/cilium-config.yaml", nil
+		}
+		trackUpgrade(&cmd, &ran)
+		verifyCiliumExecutable = func() error { return assert.AnError }
+		t.Cleanup(func() { verifyCiliumExecutable = func() error { return nil } })
+
+		require.Error(t, m.Execute(context.Background(), mctx))
+		assert.False(t, ran, "cilium upgrade must not run when the binary fails checksum verification")
+	})
 }

internal/workflows/migration_cilium_host_legacy_routing.go

@@ -121,6 +121,11 @@ func (m *CiliumHostLegacyRoutingMigration) Execute(ctx context.Context, mctx *mi
 		Str("to", "true").
 		Msg("Applying cilium-config (enable-host-legacy-routing=true) to existing cluster via 'cilium upgrade'")
 
+	// Verify the cilium binary before executing it.
+	if err := verifyCiliumExecutable(); err != nil {
+		return err
+	}
+
 	upgrade := []string{
 		fmt.Sprintf("/usr/bin/sudo %s/cilium upgrade --values %s --wait",
 			models.Paths().SandboxBinDir, configPath),

internal/workflows/migration_cilium_host_legacy_routing_test.go

@@ -46,11 +46,13 @@ func TestCiliumHostLegacyRoutingMigration_Execute(t *testing.T) {
 	origState := readHostLegacyRoutingState
 	origReconfigure := reconfigureCiliumConfig
 	origShell := runShell
+	origVerify := verifyCiliumExecutable
 	t.Cleanup(func() {
 		kubernetesInstalled = origK8s
 		readHostLegacyRoutingState = origState
 		reconfigureCiliumConfig = origReconfigure
 		runShell = origShell
+		verifyCiliumExecutable = origVerify
 	})
 
 	m := NewCiliumHostLegacyRoutingMigration()
@@ -83,8 +85,10 @@ func TestCiliumHostLegacyRoutingMigration_Execute(t *testing.T) {
 		assert.False(t, ran, "must not run before Kubernetes is installed")
 	})
 
-	// Remaining cases assume Kubernetes is present.
+	// Remaining cases assume Kubernetes is present and the cilium binary passes
+	// checksum verification unless a case overrides it.
 	kubernetesInstalled = func() bool { return true }
+	verifyCiliumExecutable = func() error { return nil }
 
 	t.Run("skips when Cilium is not installed", func(t *testing.T) {
 		var ran bool
@@ -173,4 +177,19 @@ func TestCiliumHostLegacyRoutingMigration_Execute(t *testing.T) {
 		assert.Contains(t, cmd, "--values")
 		assert.Contains(t, cmd, "--wait")
 	})
+
+	t.Run("aborts before upgrade when the cilium binary fails verification", func(t *testing.T) {
+		var ran bool
+		var cmd string
+		stateReturns(true, "", "", nil)
+		reconfigureCiliumConfig = func() (string, error) {
+			return "/opt/solo/weaver/sandbox/etc/weaver/cilium-config.yaml", nil
+		}
+		trackUpgrade(&cmd, &ran)
+		verifyCiliumExecutable = func() error { return assert.AnError }
+		t.Cleanup(func() { verifyCiliumExecutable = func() error { return nil } })
+
+		require.Error(t, m.Execute(context.Background(), mctx))
+		assert.False(t, ran, "cilium upgrade must not run when the binary fails checksum verification")
+	})
 }

internal/workflows/steps/step_cilium.go

@@ -203,6 +203,11 @@ func installCiliumCNI(version string) *automa.StepBuilder {
 		WithExecute(func(ctx context.Context, stp automa.Step) *automa.Report {
 			logx.As().Info().Str("step_id", stp.Id()).Str("software", "cilium-cni").Str("version", version).Msgf("cilium-cni version: %s", version)
 
+			// Verify the cilium binary before executing it.
+			if err := verifyExecutables("cilium"); err != nil {
+				return automa.FailureReport(stp, automa.WithError(err))
+			}
+
 			// Prepare metadata for reporting
 			meta := map[string]string{}
 
@@ -241,6 +246,11 @@ func installCiliumCNI(version string) *automa.StepBuilder {
 				return automa.SkippedReport(stp, automa.WithDetail("Cilium CNI was not installed by this step, skipping rollback"))
 			}
 
+			// Verify the cilium binary before executing it on rollback.
+			if err := verifyExecutables("cilium"); err != nil {
+				return automa.FailureReport(stp, automa.WithError(err))
+			}
+
 			// Uninstall Cilium CNI
 			scripts := []string{
 				fmt.Sprintf("/usr/bin/sudo %s/cilium uninstall", models.Paths().SandboxBinDir),

internal/workflows/steps/step_cilium_test.go

@@ -3,9 +3,12 @@
 package steps
 
 import (
+	"context"
 	"testing"
 
+	"github.com/automa-saga/automa"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 // Test_bandwidthManagerEnabled covers the decision the StartCilium guard makes on
@@ -32,3 +35,36 @@ func Test_bandwidthManagerEnabled(t *testing.T) {
 		})
 	}
 }
+
+// The install-cilium-cni step verifies the cilium binary before shelling out to
+// it on both install and rollback. These cases pin that a failed checksum aborts
+// before the binary is executed.
+
+func TestInstallCiliumCNI_FailsWhenCiliumVerificationFails(t *testing.T) {
+	calls := stubVerifyExecutables(t, func(string) error { return assert.AnError })
+
+	step, err := installCiliumCNI("1.16.5").Build()
+	require.NoError(t, err)
+
+	report := step.Execute(context.Background())
+	require.Error(t, report.Error)
+	assert.Equal(t, automa.StatusFailed, report.Status)
+	require.ErrorIs(t, report.Error, assert.AnError)
+	assert.Equal(t, []string{"cilium"}, *calls)
+}
+
+func TestInstallCiliumCNI_RollbackFailsWhenCiliumVerificationFails(t *testing.T) {
+	stubVerifyExecutables(t, func(string) error { return assert.AnError })
+
+	step, err := installCiliumCNI("1.16.5").Build()
+	require.NoError(t, err)
+
+	// Rollback is a no-op unless this step performed the install; mark it so
+	// rollback proceeds past that guard and reaches the checksum verification.
+	step.State().Local().Set(InstalledByThisStep, true)
+
+	report := step.Rollback(context.Background())
+	require.Error(t, report.Error)
+	assert.Equal(t, automa.StatusFailed, report.Status)
+	require.ErrorIs(t, report.Error, assert.AnError)
+}

internal/workflows/steps/step_kubeadm.go

@@ -186,6 +186,11 @@ func InitializeCluster() *automa.StepBuilder {
 			notify.As().StepCompletion(ctx, stp, rpt, "Kubernetes cluster initialized successfully")
 		}).
 		WithExecute(func(ctx context.Context, stp automa.Step) *automa.Report {
+			// Verify the kubectl binary before executing it.
+			if err := verifyExecutables("kubectl"); err != nil {
+				return automa.FailureReport(stp, automa.WithError(err))
+			}
+
 			// Check if cluster is already initialized
 			scripts := []string{kubectlGetNodesCmd}
 			output, err := automa_steps.RunBashScript(scripts, "")
@@ -198,6 +203,11 @@ func InitializeCluster() *automa.StepBuilder {
 
 			// Cluster not initialized, proceed with full initialization
 
+			// Verify the kubeadm binary before executing it.
+			if err := verifyExecutables("kubeadm"); err != nil {
+				return automa.FailureReport(stp, automa.WithError(err))
+			}
+
 			// Step 1: Pull kubeadm images
 			logx.As().Info().Msg("Pulling kubeadm images, this may take a while...")
 			pullImageCmd := []string{
@@ -233,6 +243,11 @@ func InitializeCluster() *automa.StepBuilder {
 			return automa.SuccessReport(stp, automa.WithDetail("Cluster initialized successfully"))
 		}).
 		WithRollback(func(ctx context.Context, stp automa.Step) *automa.Report {
+			// Verify the kubeadm binary before executing it on rollback.
+			if err := verifyExecutables("kubeadm"); err != nil {
+				return automa.FailureReport(stp, automa.WithError(err))
+			}
+
 			scripts := []string{
 				fmt.Sprintf("sudo %s/kubeadm reset --force --cri-socket unix:///opt/solo/weaver/sandbox/var/run/crio/crio.sock", models.Paths().SandboxBinDir),
 			}
@@ -260,6 +275,13 @@ func ResetCluster() *automa.StepBuilder {
 			notify.As().StepCompletion(ctx, stp, rpt, "Kubernetes cluster reset successfully")
 		}).
 		WithExecute(func(ctx context.Context, stp automa.Step) *automa.Report {
+			// Teardown is best-effort: if the kubeadm binary fails verification,
+			// skip the reset and continue rather than execute a tampered binary.
+			if err := verifyExecutables("kubeadm"); err != nil {
+				logx.As().Warn().Err(err).Msg("kubeadm checksum verification failed, skipping kubeadm reset and continuing with teardown")
+				return automa.SuccessReport(stp)
+			}
+
 			scripts := []string{
 				fmt.Sprintf("sudo %s/kubeadm reset --force --cri-socket unix:///opt/solo/weaver/sandbox/var/run/crio/crio.sock", models.Paths().SandboxBinDir),
 			}