diff --git a/Containerfile b/Containerfile index be0b228..aafe83d 100644 --- a/Containerfile +++ b/Containerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.25 AS builder +FROM golang:1.26 AS builder ARG TARGETOS ARG TARGETARCH diff --git a/cmd/main.go b/cmd/main.go index 5276216..ffd54ac 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -40,13 +41,21 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" - v1alpha1 "github.com/osac-project/bare-metal-operator/api/v1alpha1" + v1alpha1 "github.com/osac-project/bare-metal-fulfillment-operator/api/v1alpha1" "github.com/osac-project/host-management-openstack/internal/controller" "github.com/osac-project/host-management-openstack/internal/ironic" + "github.com/osac-project/osac-operator/pkg/aap" + "github.com/osac-project/osac-operator/pkg/provisioning" // +kubebuilder:scaffold:imports ) -const envHostLeaseNamespace = "HOSTLEASE_NAMESPACE" +const ( + envHostLeaseNamespace = "HOSTLEASE_NAMESPACE" + envAAPURL = "OSAC_AAP_URL" + envAAPToken = "OSAC_AAP_TOKEN" + envAAPInsecureSkip = "OSAC_AAP_INSECURE_SKIP_VERIFY" + envAAPTemplatePrefix = "OSAC_AAP_TEMPLATE_PREFIX" +) var ( scheme = runtime.NewScheme() @@ -230,11 +239,37 @@ func main() { } setupLog.Info("Connect to ironic", "endpoint", ironicClient.GetEndpoint()) + // AAP provisioning provider for image provisioning workflows + var provisioningProvider provisioning.ProvisioningProvider + if aapURL := os.Getenv(envAAPURL); aapURL != "" { + aapToken := os.Getenv(envAAPToken) + insecureSkipVerify, _ := strconv.ParseBool(os.Getenv(envAAPInsecureSkip)) + templatePrefix := os.Getenv(envAAPTemplatePrefix) + if templatePrefix == "" { + templatePrefix = "osac" + } + aapClient := aap.NewClient(aapURL, aapToken, insecureSkipVerify) + var err error + provisioningProvider, err = provisioning.NewProvider(provisioning.ProviderConfig{ + ProviderType: provisioning.ProviderTypeAAP, + AAPClient: aapClient, + TemplatePrefix: templatePrefix, + }) + if err != nil { + setupLog.Error(err, "failed to create provisioning provider") + os.Exit(1) + } + setupLog.Info("AAP provisioning provider configured", "url", aapURL, "templatePrefix", templatePrefix) + } else { + setupLog.Info("AAP not configured, provisioning workflows disabled", "envVar", envAAPURL) + } + // Create HostLease reconciler with defaults hostLeaseReconciler := controller.NewHostLeaseReconciler( mgr.GetClient(), mgr.GetScheme(), ironicClient, + provisioningProvider, 0, // Use DefaultRecheckInterval ) if err := hostLeaseReconciler.SetupWithManager(mgr); err != nil { diff --git a/cmd/main_test.go b/cmd/main_test.go index d6ee2e1..9ba59d6 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -22,7 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" //nolint:revive,staticcheck . "github.com/onsi/gomega" //nolint:revive,staticcheck - osacv1alpha1 "github.com/osac-project/bare-metal-operator/api/v1alpha1" + osacv1alpha1 "github.com/osac-project/bare-metal-fulfillment-operator/api/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" diff --git a/config/samples/v1alpha1_hostlease.yaml b/config/samples/v1alpha1_hostlease.yaml index 7c03768..9f4a066 100644 --- a/config/samples/v1alpha1_hostlease.yaml +++ b/config/samples/v1alpha1_hostlease.yaml @@ -7,10 +7,11 @@ spec: externalHostID: efa3d310-164d-44a4-a399-46cbcdaa587a externalHostName: MOC-R4PAC08U21-S1 hostType: fc830 - # hostClass: openstack + hostClass: openstack networkClass: openstack networkInterfaces: - macAddress: aa:bb:cc:dd:ee:f1 network: private-vlan-network # poweredOn: on - templateID: default_template + templateID: bm_host_agent_provisioning + templateParameters: '{"provisioningState":"active","imageSpec":{"url":"xxx"},"provisioningNetwork":"idle-agents-network"}' diff --git a/go.mod b/go.mod index d22cf7d..840d3b3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/osac-project/host-management-openstack -go 1.25.0 +go 1.26.0 require ( github.com/go-logr/logr v1.4.3 @@ -8,6 +8,8 @@ require ( github.com/gophercloud/utils/v2 v2.0.0-20260409164906-cd2167a8681a github.com/onsi/ginkgo/v2 v2.28.1 github.com/onsi/gomega v1.39.1 + github.com/osac-project/osac-operator v0.1.1-0.20260511193951-8bf9632098a0 + github.com/osac-project/osac-operator/api v0.0.2-0.20260511193951-8bf9632098a0 k8s.io/apimachinery v0.35.4 k8s.io/client-go v0.35.4 sigs.k8s.io/controller-runtime v0.23.3 @@ -16,7 +18,6 @@ require ( require ( github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/gofrs/uuid/v5 v5.4.0 // indirect - github.com/osac-project/osac-operator v0.0.0-20260413152945-199a0469a4c2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect @@ -57,7 +58,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/osac-project/bare-metal-operator v0.0.0-20260423171233-28ec2bd86640 + github.com/osac-project/bare-metal-fulfillment-operator v0.0.0-20260518182443-35f2a9590dcc github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect @@ -80,7 +81,7 @@ require ( golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac // indirect golang.org/x/net v0.49.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.19.0 // indirect + golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/term v0.39.0 // indirect golang.org/x/text v0.33.0 // indirect @@ -96,8 +97,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.35.4 k8s.io/apiextensions-apiserver v0.35.0 // indirect - k8s.io/apiserver v0.35.0 // indirect - k8s.io/component-base v0.35.0 // indirect + k8s.io/apiserver v0.35.4 // indirect + k8s.io/component-base v0.35.4 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect diff --git a/go.sum b/go.sum index 6276d26..c162036 100644 --- a/go.sum +++ b/go.sum @@ -111,10 +111,12 @@ github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= -github.com/osac-project/bare-metal-operator v0.0.0-20260423171233-28ec2bd86640 h1:dYT1/qh3zVJGryZO+ySdLCQNmCkXCzl527Bd0m4bs8g= -github.com/osac-project/bare-metal-operator v0.0.0-20260423171233-28ec2bd86640/go.mod h1:QifCZ5ITT0CLDQ1ZaG0ac9w+9KAmtfnafesvxZ635bM= -github.com/osac-project/osac-operator v0.0.0-20260413152945-199a0469a4c2 h1:DI1B9fMQgjUlkHVL7OuO1fQ24MmYQIrktKslqp0K9nE= -github.com/osac-project/osac-operator v0.0.0-20260413152945-199a0469a4c2/go.mod h1:k/5CPGjhIS9n/BRBQHtxABzeA69yjtBWtLHmnloYeqE= +github.com/osac-project/bare-metal-fulfillment-operator v0.0.0-20260518182443-35f2a9590dcc h1:Ayw/y3WiCxQh8I8nKuyacNxrBeq1GCw60bNLfFD16rQ= +github.com/osac-project/bare-metal-fulfillment-operator v0.0.0-20260518182443-35f2a9590dcc/go.mod h1:N+t/4UPZajrHeDSjKAy+PwyydVaiONx4Uv0DRhk6o0I= +github.com/osac-project/osac-operator v0.1.1-0.20260511193951-8bf9632098a0 h1:w2POmUbIU+ecHGUky0SifYFWV5W05VijjixjnpWN2xw= +github.com/osac-project/osac-operator v0.1.1-0.20260511193951-8bf9632098a0/go.mod h1:c207O7XvQtIY6hFIqUAv3yB2JQt1ZRrsWA+wEla87aQ= +github.com/osac-project/osac-operator/api v0.0.2-0.20260511193951-8bf9632098a0 h1:ddxVNqBMGOk/xx19OgAuLZpKpId7A14pElhQ6eslAl8= +github.com/osac-project/osac-operator/api v0.0.2-0.20260511193951-8bf9632098a0/go.mod h1:JBz/wSWPT4NhksjXZs2QUjulga+KTUdW5UJX0kB22aA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -197,8 +199,8 @@ golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= @@ -237,12 +239,12 @@ k8s.io/apiextensions-apiserver v0.35.0 h1:3xHk2rTOdWXXJM+RDQZJvdx0yEOgC0FgQ1PlJa k8s.io/apiextensions-apiserver v0.35.0/go.mod h1:E1Ahk9SADaLQ4qtzYFkwUqusXTcaV2uw3l14aqpL2LU= k8s.io/apimachinery v0.35.4 h1:xtdom9RG7e+yDp71uoXoJDWEE2eOiHgeO4GdBzwWpds= k8s.io/apimachinery v0.35.4/go.mod h1:NNi1taPOpep0jOj+oRha3mBJPqvi0hGdaV8TCqGQ+cc= -k8s.io/apiserver v0.35.0 h1:CUGo5o+7hW9GcAEF3x3usT3fX4f9r8xmgQeCBDaOgX4= -k8s.io/apiserver v0.35.0/go.mod h1:QUy1U4+PrzbJaM3XGu2tQ7U9A4udRRo5cyxkFX0GEds= +k8s.io/apiserver v0.35.4 h1:vtuFqNFmF9bPRdHDL2lpK6qCTPWDreZJL4LRPwVM6ho= +k8s.io/apiserver v0.35.4/go.mod h1:JnBcb+J8kFXKpZkgcbcUnPBBHi4qgBii1I7dLxFY/oo= k8s.io/client-go v0.35.4 h1:DN6fyaGuzK64UvnKO5fOA6ymSjvfGAnCAHAR0C66kD8= k8s.io/client-go v0.35.4/go.mod h1:2Pg9WpsS4NeOpoYTfHHfMxBG8zFMSAUi4O/qoiJC3nY= -k8s.io/component-base v0.35.0 h1:+yBrOhzri2S1BVqyVSvcM3PtPyx5GUxCK2tinZz1G94= -k8s.io/component-base v0.35.0/go.mod h1:85SCX4UCa6SCFt6p3IKAPej7jSnF3L8EbfSyMZayJR0= +k8s.io/component-base v0.35.4 h1:6n1tNJ87johN0Hif0Fs8K2GMthsaUwMqCebUDLYyv7U= +k8s.io/component-base v0.35.4/go.mod h1:qaDJgz5c1KYKla9occFmlJEfPpkuA55s90G509R+PeY= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= diff --git a/internal/controller/hostlease_controller.go b/internal/controller/hostlease_controller.go index efae643..e5b8ada 100644 --- a/internal/controller/hostlease_controller.go +++ b/internal/controller/hostlease_controller.go @@ -20,10 +20,12 @@ package controller import ( "context" "errors" + "fmt" "time" "github.com/go-logr/logr" "github.com/gophercloud/gophercloud/v2/openstack/baremetal/v1/nodes" + "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -34,18 +36,24 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" - v1alpha1 "github.com/osac-project/bare-metal-operator/api/v1alpha1" + v1alpha1 "github.com/osac-project/bare-metal-fulfillment-operator/api/v1alpha1" "github.com/osac-project/host-management-openstack/internal/ironic" + opv1alpha1 "github.com/osac-project/osac-operator/api/v1alpha1" + "github.com/osac-project/osac-operator/pkg/provisioning" ) // HostLeaseReconciler reconciles HostLease CRs for power management via Ironic. type HostLeaseReconciler struct { client.Client - Scheme *runtime.Scheme - IronicClient ironic.NodeClient + Scheme *runtime.Scheme + IronicClient ironic.NodeClient + ProvisioningProvider provisioning.ProvisioningProvider // RecheckInterval is the interval for polling Ironic until power state matches desired state. RecheckInterval time.Duration + + // ProvisionPollInterval is the interval for polling AAP job status. + ProvisionPollInterval time.Duration } // NewHostLeaseReconciler creates a new HostLeaseReconciler with defaults applied. @@ -53,6 +61,7 @@ func NewHostLeaseReconciler( client client.Client, scheme *runtime.Scheme, ironicClient ironic.NodeClient, + provider provisioning.ProvisioningProvider, recheckInterval time.Duration, ) *HostLeaseReconciler { if recheckInterval <= 0 { @@ -60,10 +69,12 @@ func NewHostLeaseReconciler( } return &HostLeaseReconciler{ - Client: client, - Scheme: scheme, - IronicClient: ironicClient, - RecheckInterval: recheckInterval, + Client: client, + Scheme: scheme, + IronicClient: ironicClient, + ProvisioningProvider: provider, + RecheckInterval: recheckInterval, + ProvisionPollInterval: DefaultProvisionPollInterval, } } @@ -79,15 +90,36 @@ func (r *HostLeaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, client.IgnoreNotFound(err) } + oldstatus := hostLease.Status.DeepCopy() + + var result ctrl.Result + var err error if !hostLease.DeletionTimestamp.IsZero() { - return r.reconcileDelete(ctx, hostLease, log) + result, err = r.reconcileDelete(ctx, hostLease, log) + } else { + result, err = r.handleUpdate(ctx, hostLease, log) } - // Check if the HostLease should be managed by this controller + if !equality.Semantic.DeepEqual(hostLease.Status, *oldstatus) { + log.Info("Persisting HostLease status changes", "hostLease", hostLease.Name) + if statusErr := r.Status().Update(ctx, hostLease); client.IgnoreNotFound(statusErr) != nil { + log.Error(statusErr, "failed to update HostLease status") + return result, statusErr + } + } + + return result, err +} + +func (r *HostLeaseReconciler) handleUpdate(ctx context.Context, hostLease *v1alpha1.HostLease, log logr.Logger) (ctrl.Result, error) { if !r.validateOpenStackHost(hostLease, log) { return ctrl.Result{}, nil } + if hostLease.Status.Phase == "" { + hostLease.Status.Phase = v1alpha1.HostLeasePhaseProgressing + } + if !controllerutil.ContainsFinalizer(hostLease, hostLeaseFinalizer) { controllerutil.AddFinalizer(hostLease, hostLeaseFinalizer) if err := r.Update(ctx, hostLease); err != nil { @@ -96,39 +128,41 @@ func (r *HostLeaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{Requeue: true}, nil } + // Provisioning runs first — power reconciliation is suspended during provisioning + if r.ProvisioningProvider != nil && hostLease.Spec.TemplateID != "" && hostLease.Spec.TemplateID != "noop" { + result, provErr := r.reconcileProvisioning(ctx, hostLease) + if provErr != nil { + hostLease.Status.Phase = v1alpha1.HostLeasePhaseFailed + return result, provErr + } + if !result.IsZero() { + return result, nil + } + } + node, err := r.IronicClient.GetNode(ctx, hostLease.Spec.ExternalHostID) if err != nil { log.Error(err, "failed to get Ironic node", "nodeID", hostLease.Spec.ExternalHostID) - if statusErr := r.syncHostLeaseStatus(ctx, hostLease, nil, err, log); statusErr != nil { - log.Error(statusErr, "failed to update HostLease status after get node failure") - } + r.syncHostLeaseStatus(hostLease, nil, err, log) return ctrl.Result{}, err } log.V(1).Info("Ironic node", "nodeID", hostLease.Spec.ExternalHostID, "power_state", node.PowerState) - // If Spec.PoweredOn is set, call ironic to set the power. Otherwise, just update the status. if hostLease.Spec.PoweredOn != nil { if err := r.reconcilePower(ctx, hostLease, node, log); err != nil { - if statusErr := r.syncHostLeaseStatus(ctx, hostLease, nil, err, log); statusErr != nil { - log.Error(statusErr, "failed to update HostLease status after power failure") - } + r.syncHostLeaseStatus(hostLease, nil, err, log) return ctrl.Result{}, err } node, err = r.IronicClient.GetNode(ctx, hostLease.Spec.ExternalHostID) if err != nil { log.Error(err, "failed to refresh node after power reconciliation", "nodeID", hostLease.Spec.ExternalHostID) - if statusErr := r.syncHostLeaseStatus(ctx, hostLease, nil, err, log); statusErr != nil { - log.Error(statusErr, "failed to update HostLease status after node refresh failure") - } + r.syncHostLeaseStatus(hostLease, nil, err, log) return ctrl.Result{}, err } } - if err := r.syncHostLeaseStatus(ctx, hostLease, node, nil, log); err != nil { - log.Error(err, "failed to update HostLease status") - return ctrl.Result{}, err - } + r.syncHostLeaseStatus(hostLease, node, nil, log) if hostLease.Spec.PoweredOn != nil { currentlyOn := node.PowerState == ironic.PowerOn.String() @@ -137,6 +171,8 @@ func (r *HostLeaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } } + hostLease.Status.Phase = v1alpha1.HostLeasePhaseReady + log.Info("HostLease reconcile completed; status changes pending persistence", "hostLease", hostLease.Name) return ctrl.Result{}, nil } @@ -155,9 +191,15 @@ func (r *HostLeaseReconciler) reconcileDelete(ctx context.Context, hostLease *v1 } hostLease.Status.Phase = v1alpha1.HostLeasePhaseDeleting - if err := r.Status().Update(ctx, hostLease); err != nil { - log.Error(err, "failed to update HostLease status phase to Deleting") - return ctrl.Result{}, err + + if r.ProvisioningProvider != nil && hostLease.Spec.TemplateID != "" && hostLease.Spec.TemplateID != "noop" { + result, done, err := r.reconcileDeprovisioning(ctx, hostLease) + if err != nil { + return result, err + } + if !done { + return result, nil + } } log.Info("Unsetting hostClass and removing finalizer") @@ -171,6 +213,71 @@ func (r *HostLeaseReconciler) reconcileDelete(ctx context.Context, hostLease *v1 return ctrl.Result{}, nil } +func (r *HostLeaseReconciler) reconcileDeprovisioning(ctx context.Context, hostLease *v1alpha1.HostLease) (ctrl.Result, bool, error) { + if hostLease.Status.Jobs == nil { + hostLease.Status.Jobs = []opv1alpha1.JobStatus{} + } + + latestDeprovisionJob := provisioning.FindLatestJobByType(hostLease.Status.Jobs, opv1alpha1.JobTypeDeprovision) + + if !provisioning.HasJobID(latestDeprovisionJob) { + result, err := provisioning.TriggerDeprovisionJob( + ctx, r.ProvisioningProvider, hostLease, + &hostLease.Status.Jobs, provisioning.DefaultMaxJobHistory, r.ProvisionPollInterval, + ) + if err != nil { + hostLease.SetStatusCondition( + v1alpha1.HostConditionDeprovisionTemplateComplete, + metav1.ConditionFalse, + v1alpha1.HostConditionReasonTemplateFailed, + err.Error(), + ) + return result, false, err + } + if err := r.Status().Update(ctx, hostLease); err != nil { + return ctrl.Result{}, false, fmt.Errorf("failed to flush status after deprovision trigger: %w", err) + } + if !result.IsZero() { + hostLease.SetStatusCondition( + v1alpha1.HostConditionDeprovisionTemplateComplete, + metav1.ConditionFalse, + v1alpha1.HostConditionReasonProgressing, + "Deprovision job in progress", + ) + return result, false, nil + } + return ctrl.Result{}, true, nil + } + + result, done, err := provisioning.PollDeprovisionJob( + ctx, r.ProvisioningProvider, hostLease, + &hostLease.Status.Jobs, latestDeprovisionJob, r.ProvisionPollInterval, + ) + if err != nil { + return result, false, err + } + + if done { + if latestDeprovisionJob.State.IsSuccessful() { + hostLease.SetStatusCondition( + v1alpha1.HostConditionDeprovisionTemplateComplete, + metav1.ConditionTrue, + "Succeeded", + "Deprovision job completed successfully", + ) + } + } else { + hostLease.SetStatusCondition( + v1alpha1.HostConditionDeprovisionTemplateComplete, + metav1.ConditionFalse, + v1alpha1.HostConditionReasonProgressing, + "Deprovision job in progress", + ) + } + + return result, done, nil +} + func (r *HostLeaseReconciler) validateOpenStackHost(hostLease *v1alpha1.HostLease, log logr.Logger) bool { if hostLease.Spec.ExternalHostID == "" { log.V(1).Info("HostLease skipped", "reason", "spec.externalHostID not set") @@ -219,8 +326,64 @@ func (r *HostLeaseReconciler) reconcilePower(ctx context.Context, hostLease *v1a return err } -// syncHostLeaseStatus syncs the phase, conditions, power state and updates the HostLease status. -func (r *HostLeaseReconciler) syncHostLeaseStatus(ctx context.Context, hostLease *v1alpha1.HostLease, node *nodes.Node, reconcileErr error, log logr.Logger) error { +func (r *HostLeaseReconciler) reconcileProvisioning(ctx context.Context, hostLease *v1alpha1.HostLease) (ctrl.Result, error) { + desiredVersion, err := provisioning.ComputeDesiredConfigVersion(hostLease.Spec) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to compute desired config version: %w", err) + } + hostLease.Status.DesiredConfigVersion = desiredVersion + + result, err := provisioning.RunProvisioningLifecycle(ctx, r.ProvisioningProvider, hostLease, + &provisioning.State{Jobs: &hostLease.Status.Jobs, DesiredConfigVersion: desiredVersion}, + provisioning.DefaultMaxJobHistory, r.ProvisionPollInterval, + &provisioning.PollCallbacks{ + OnFailed: func(message string) { + hostLease.Status.Phase = v1alpha1.HostLeasePhaseFailed + hostLease.SetStatusCondition( + v1alpha1.HostConditionProvisionTemplateComplete, + metav1.ConditionFalse, + v1alpha1.HostConditionReasonTemplateFailed, + message, + ) + }, + OnSuccess: func(_ provisioning.ProvisionStatus) { + hostLease.SetStatusCondition( + v1alpha1.HostConditionProvisionTemplateComplete, + metav1.ConditionTrue, + "Succeeded", + "Provision job completed successfully", + ) + }, + }, + func() bool { + return provisioning.CheckAPIServerForNonTerminalProvisionJob( + ctx, r.Client, client.ObjectKeyFromObject(hostLease), &v1alpha1.HostLease{}, + ) + }, + func() error { + return r.Status().Update(ctx, hostLease) + }, + ) + if err != nil { + return result, err + } + + // Set progressing condition while provisioning is in-flight, but don't overwrite a failure. + provisionCond := hostLease.GetStatusCondition(v1alpha1.HostConditionProvisionTemplateComplete) + if result.RequeueAfter > 0 && (provisionCond == nil || provisionCond.Reason != v1alpha1.HostConditionReasonTemplateFailed) { + hostLease.SetStatusCondition( + v1alpha1.HostConditionProvisionTemplateComplete, + metav1.ConditionFalse, + v1alpha1.HostConditionReasonProgressing, + "Provisioning job in progress", + ) + } + + return result, nil +} + +// syncHostLeaseStatus syncs power-related conditions and observed power state in memory. +func (r *HostLeaseReconciler) syncHostLeaseStatus(hostLease *v1alpha1.HostLease, node *nodes.Node, reconcileErr error, log logr.Logger) { if reconcileErr != nil { hostLease.Status.Phase = v1alpha1.HostLeasePhaseFailed hostLease.SetStatusCondition( @@ -229,41 +392,33 @@ func (r *HostLeaseReconciler) syncHostLeaseStatus(ctx context.Context, hostLease v1alpha1.HostConditionReasonIronicAPIFailure, reconcileErr.Error(), ) - log.Info("Updating HostLease status", "phase", hostLease.Status.Phase, "condition", v1alpha1.HostConditionPowerSynced, "conditionStatus", metav1.ConditionFalse, "reason", v1alpha1.HostConditionReasonIronicAPIFailure) - return r.Status().Update(ctx, hostLease) + log.Info("HostLease status synced", "phase", hostLease.Status.Phase, "condition", v1alpha1.HostConditionPowerSynced, "conditionStatus", metav1.ConditionFalse, "reason", v1alpha1.HostConditionReasonIronicAPIFailure) + return } if node == nil { - return nil + return } poweredOn := node.PowerState == ironic.PowerOn.String() hostLease.Status.PoweredOn = &poweredOn if hostLease.Spec.PoweredOn != nil && *hostLease.Spec.PoweredOn != poweredOn { - hostLease.Status.Phase = v1alpha1.HostLeasePhaseProgressing hostLease.SetStatusCondition( v1alpha1.HostConditionPowerSynced, metav1.ConditionFalse, v1alpha1.HostConditionReasonProgressing, "waiting for node power state to converge", ) - return r.Status().Update(ctx, hostLease) - } else { - hostLease.Status.Phase = v1alpha1.HostLeasePhaseReady - } - - if poweredOn { + } else if poweredOn { hostLease.SetStatusCondition(v1alpha1.HostConditionPowerSynced, metav1.ConditionTrue, v1alpha1.HostConditionReasonPowerOn, "") - log.Info("Updating HostLease power status", "phase", hostLease.Status.Phase, "poweredOn", poweredOn, "condition", v1alpha1.HostConditionPowerSynced, "conditionStatus", metav1.ConditionTrue, "reason", v1alpha1.HostConditionReasonPowerOn) + log.Info("HostLease power status synced", "poweredOn", poweredOn, "condition", v1alpha1.HostConditionPowerSynced, "conditionStatus", metav1.ConditionTrue, "reason", v1alpha1.HostConditionReasonPowerOn) } else { hostLease.SetStatusCondition(v1alpha1.HostConditionPowerSynced, metav1.ConditionTrue, v1alpha1.HostConditionReasonPowerOff, "") - log.Info("Updating HostLease power status", "phase", hostLease.Status.Phase, "poweredOn", poweredOn, "condition", v1alpha1.HostConditionPowerSynced, "conditionStatus", metav1.ConditionTrue, "reason", v1alpha1.HostConditionReasonPowerOff) + log.Info("HostLease power status synced", "poweredOn", poweredOn, "condition", v1alpha1.HostConditionPowerSynced, "conditionStatus", metav1.ConditionTrue, "reason", v1alpha1.HostConditionReasonPowerOff) } - - return r.Status().Update(ctx, hostLease) } func hostLeasePredicate() predicate.Funcs { diff --git a/internal/controller/hostlease_controller_test.go b/internal/controller/hostlease_controller_test.go index 3e57a52..32f33ef 100644 --- a/internal/controller/hostlease_controller_test.go +++ b/internal/controller/hostlease_controller_test.go @@ -36,8 +36,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/predicate" - v1alpha1 "github.com/osac-project/bare-metal-operator/api/v1alpha1" + v1alpha1 "github.com/osac-project/bare-metal-fulfillment-operator/api/v1alpha1" "github.com/osac-project/host-management-openstack/internal/ironic" + opv1alpha1 "github.com/osac-project/osac-operator/api/v1alpha1" + "github.com/osac-project/osac-operator/pkg/provisioning" logf "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -47,6 +49,7 @@ const ( testPowerOn = "power on" testNodeID = "node-1" testNamespace = "default" + testNoopTmpl = "noop" ) // mockIronicClient implements ironic.NodeClient for testing. @@ -104,14 +107,20 @@ var _ = Describe("HostLeaseReconciler", func() { Describe("NewHostLeaseReconciler", func() { It("should use the provided recheck interval when positive", func() { - r := NewHostLeaseReconciler(nil, testScheme, mockIronic, 30*time.Second) + r := NewHostLeaseReconciler(nil, testScheme, mockIronic, nil, 30*time.Second) Expect(r.RecheckInterval).To(Equal(30 * time.Second)) }) It("should use the default recheck interval when zero", func() { - r := NewHostLeaseReconciler(nil, testScheme, mockIronic, 0) + r := NewHostLeaseReconciler(nil, testScheme, mockIronic, nil, 0) Expect(r.RecheckInterval).To(Equal(DefaultRecheckInterval)) }) + + It("should store the provisioning provider", func() { + mockProvider := &provisioning.AAPProvider{} + r := NewHostLeaseReconciler(nil, testScheme, mockIronic, mockProvider, 0) + Expect(r.ProvisioningProvider).To(Equal(mockProvider)) + }) }) Describe("validateOpenStackHost", func() { @@ -679,6 +688,155 @@ var _ = Describe("HostLeaseReconciler", func() { }) }) + Describe("reconcileProvisioning", func() { + It("should skip provisioning when ProvisioningProvider is nil", func() { + hostLease := &v1alpha1.HostLease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hostlease-no-aap", + Namespace: testNamespace, + Finalizers: []string{hostLeaseFinalizer}, + }, + Spec: v1alpha1.HostLeaseSpec{ + ExternalHostID: testNodeID, + HostClass: hostClass, + TemplateID: "image-provision", + }, + } + reconciler.Client = fake.NewClientBuilder(). + WithScheme(testScheme). + WithStatusSubresource(hostLease). + WithObjects(hostLease). + Build() + reconciler.ProvisioningProvider = nil + + mockIronic.getNodeFunc = func(_ context.Context, _ string) (*nodes.Node, error) { + return &nodes.Node{PowerState: ironic.PowerOff.String()}, nil + } + + result, err := reconciler.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: hostLease.Name, + Namespace: hostLease.Namespace, + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + }) + + It("should skip provisioning when templateID is noop", func() { + hostLease := &v1alpha1.HostLease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hostlease-noop", + Namespace: testNamespace, + Finalizers: []string{hostLeaseFinalizer}, + }, + Spec: v1alpha1.HostLeaseSpec{ + ExternalHostID: testNodeID, + HostClass: hostClass, + TemplateID: testNoopTmpl, + }, + } + reconciler.ProvisioningProvider = &provisioning.AAPProvider{} + reconciler.Client = fake.NewClientBuilder(). + WithScheme(testScheme). + WithStatusSubresource(hostLease). + WithObjects(hostLease). + Build() + + mockIronic.getNodeFunc = func(_ context.Context, _ string) (*nodes.Node, error) { + return &nodes.Node{PowerState: ironic.PowerOff.String()}, nil + } + + result, err := reconciler.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: hostLease.Name, + Namespace: hostLease.Namespace, + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + }) + + It("should skip provisioning when templateID is empty", func() { + hostLease := &v1alpha1.HostLease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hostlease-empty-template", + Namespace: testNamespace, + Finalizers: []string{hostLeaseFinalizer}, + }, + Spec: v1alpha1.HostLeaseSpec{ + ExternalHostID: testNodeID, + HostClass: hostClass, + TemplateID: "", + }, + } + reconciler.ProvisioningProvider = &provisioning.AAPProvider{} + reconciler.Client = fake.NewClientBuilder(). + WithScheme(testScheme). + WithStatusSubresource(hostLease). + WithObjects(hostLease). + Build() + + mockIronic.getNodeFunc = func(_ context.Context, _ string) (*nodes.Node, error) { + return &nodes.Node{PowerState: ironic.PowerOff.String()}, nil + } + + result, err := reconciler.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: hostLease.Name, + Namespace: hostLease.Namespace, + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + }) + + It("should not re-trigger when a successful provision job exists", func() { + hostLease := &v1alpha1.HostLease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hostlease-already-provisioned", + Namespace: testNamespace, + Finalizers: []string{hostLeaseFinalizer}, + }, + Spec: v1alpha1.HostLeaseSpec{ + ExternalHostID: testNodeID, + HostClass: hostClass, + TemplateID: "image-provision", + }, + Status: v1alpha1.HostLeaseStatus{ + Jobs: []opv1alpha1.JobStatus{ + { + JobID: "123", + Type: opv1alpha1.JobTypeProvision, + State: opv1alpha1.JobStateSucceeded, + Message: "successful", + Timestamp: metav1.Now(), + }, + }, + }, + } + reconciler.ProvisioningProvider = &provisioning.AAPProvider{} + reconciler.Client = fake.NewClientBuilder(). + WithScheme(testScheme). + WithStatusSubresource(hostLease). + WithObjects(hostLease). + Build() + + mockIronic.getNodeFunc = func(_ context.Context, _ string) (*nodes.Node, error) { + return &nodes.Node{PowerState: ironic.PowerOff.String()}, nil + } + + result, err := reconciler.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: hostLease.Name, + Namespace: hostLease.Namespace, + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + }) + }) + Describe("syncHostLeaseStatus", func() { var hostLease *v1alpha1.HostLease @@ -700,11 +858,9 @@ var _ = Describe("HostLeaseReconciler", func() { Build() }) - It("should set phase to Failed and PowerSynced to False on error", func() { - err := reconciler.syncHostLeaseStatus(context.Background(), hostLease, nil, errors.New("ironic connection failed"), log) - Expect(err).NotTo(HaveOccurred()) + It("should set PowerSynced to False on error", func() { + reconciler.syncHostLeaseStatus(hostLease, nil, errors.New("ironic connection failed"), log) - Expect(hostLease.Status.Phase).To(Equal(v1alpha1.HostLeasePhaseFailed)) condition := hostLease.GetStatusCondition(v1alpha1.HostConditionPowerSynced) Expect(condition).NotTo(BeNil()) Expect(condition.Status).To(Equal(metav1.ConditionFalse)) @@ -712,12 +868,10 @@ var _ = Describe("HostLeaseReconciler", func() { Expect(condition.Message).To(Equal("ironic connection failed")) }) - It("should set phase to Ready and PowerSynced to True when node is on", func() { + It("should set PowerSynced to True when node is on", func() { node := &nodes.Node{PowerState: testPowerOn} - err := reconciler.syncHostLeaseStatus(context.Background(), hostLease, node, nil, log) - Expect(err).NotTo(HaveOccurred()) + reconciler.syncHostLeaseStatus(hostLease, node, nil, log) - Expect(hostLease.Status.Phase).To(Equal(v1alpha1.HostLeasePhaseReady)) Expect(hostLease.Status.PoweredOn).NotTo(BeNil()) Expect(*hostLease.Status.PoweredOn).To(BeTrue()) @@ -727,12 +881,10 @@ var _ = Describe("HostLeaseReconciler", func() { Expect(condition.Reason).To(Equal(v1alpha1.HostConditionReasonPowerOn)) }) - It("should set phase to Ready and PowerSynced to True when node is off", func() { + It("should set PowerSynced to True when node is off", func() { node := &nodes.Node{PowerState: testPowerOff} - err := reconciler.syncHostLeaseStatus(context.Background(), hostLease, node, nil, log) - Expect(err).NotTo(HaveOccurred()) + reconciler.syncHostLeaseStatus(hostLease, node, nil, log) - Expect(hostLease.Status.Phase).To(Equal(v1alpha1.HostLeasePhaseReady)) Expect(hostLease.Status.PoweredOn).NotTo(BeNil()) Expect(*hostLease.Status.PoweredOn).To(BeFalse()) @@ -742,13 +894,11 @@ var _ = Describe("HostLeaseReconciler", func() { Expect(condition.Reason).To(Equal(v1alpha1.HostConditionReasonPowerOff)) }) - It("should set phase to Progressing when power state does not match desired", func() { + It("should set PowerSynced to False when power state does not match desired", func() { hostLease.Spec.PoweredOn = boolPtr(true) node := &nodes.Node{PowerState: testPowerOff} - err := reconciler.syncHostLeaseStatus(context.Background(), hostLease, node, nil, log) - Expect(err).NotTo(HaveOccurred()) + reconciler.syncHostLeaseStatus(hostLease, node, nil, log) - Expect(hostLease.Status.Phase).To(Equal(v1alpha1.HostLeasePhaseProgressing)) Expect(hostLease.Status.PoweredOn).NotTo(BeNil()) Expect(*hostLease.Status.PoweredOn).To(BeFalse()) condition := hostLease.GetStatusCondition(v1alpha1.HostConditionPowerSynced) @@ -758,8 +908,7 @@ var _ = Describe("HostLeaseReconciler", func() { }) It("should not modify status when node is nil and no error", func() { - err := reconciler.syncHostLeaseStatus(context.Background(), hostLease, nil, nil, log) - Expect(err).NotTo(HaveOccurred()) + reconciler.syncHostLeaseStatus(hostLease, nil, nil, log) Expect(hostLease.Status.PoweredOn).To(BeNil()) Expect(hostLease.Status.Conditions).To(BeEmpty()) diff --git a/internal/controller/hostlease_names.go b/internal/controller/hostlease_names.go index 526a8f9..11aee66 100644 --- a/internal/controller/hostlease_names.go +++ b/internal/controller/hostlease_names.go @@ -24,4 +24,7 @@ const ( // DefaultRecheckInterval is the default interval for polling Ironic power state. DefaultRecheckInterval = 60 * time.Second + + // DefaultProvisionPollInterval is the default interval for polling AAP job status. + DefaultProvisionPollInterval = 30 * time.Second )