diff --git a/.gitignore b/.gitignore index ad47527..09a152c 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,11 @@ go.work.sum # Editor/IDE .idea/ .vscode/ + +# Worktrees +.worktrees/ + +# Superpowers plans (local planning artifacts) +docs/superpowers/plans/ +docs/superpowers/ +.envrc diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..ac3095f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,317 @@ +# kube-xset Project Guide + +kube-xset is a Kubernetes utility package for managing operations (scaling, upgrading, replacing) on a set of Kubernetes resources. It provides a reusable framework for building controllers that manage workload sets with advanced lifecycle management. + +## Project Structure + +``` +kube-xset/ +├── api/ # Core API definitions and interfaces +│ ├── xset_controller_types.go # XSetController interface (main entry point) +│ ├── xset_types.go # XSetSpec, XSetStatus, update/scale strategies +│ ├── resourcecontext_types.go # ResourceContext for ID allocation +│ ├── well_knowns.go # Label/annotation constants +│ └── validation/ # Validation helpers +├── synccontrols/ # Core sync logic (Scale, Update, Replace) +│ ├── sync_control.go # Main SyncControl interface +│ ├── x_scale.go # Scaling logic +│ ├── x_update.go # Update logic +│ ├── x_replace.go # Replace logic +│ └── inexclude.go # Include/exclude targets +├── resourcecontexts/ # ResourceContext management (ID allocation) +├── opslifecycle/ # Ops lifecycle management (graceful operations) +├── xcontrol/ # Target control helpers +├── subresources/ # Subresource management (PVC) +├── revisionowner/ # Revision ownership tracking +├── features/ # Feature gates +└── xset_controller.go # Main SetUpWithManager function +``` + +## How to Use kube-xset + +### 1. Implement XSetController Interface + +The main entry point is implementing the `XSetController` interface from `api/xset_controller_types.go`: + +```go +type XSetController interface { + ControllerName() string + FinalizerName() string + + XSetMeta() metav1.TypeMeta // GVK for XSet (e.g., CollaSet) + XMeta() metav1.TypeMeta // GVK for X (e.g., Pod) + NewXSetObject() XSetObject // Constructor for XSet + NewXObject() client.Object // Constructor for X + NewXObjectList() client.ObjectList + + // Required interfaces + XSetOperation // Access XSet spec/status + XOperation // Access X object and status + + // Optional interfaces (implement as needed) + // - LifecycleAdapterGetter + // - ResourceContextAdapterGetter + // - LabelAnnotationManagerGetter + // - SubResourcePvcAdapter + // - SubResourceAdapterGetter + // - DecorationAdapter +} +``` + +### 2. Implement Required Interfaces + +#### XSetOperation Interface + +```go +type XSetOperation interface { + GetXSetSpec(object XSetObject) *XSetSpec + GetXSetPatch(object metav1.Object) ([]byte, error) + GetXSetStatus(object XSetObject) *XSetStatus + SetXSetStatus(object XSetObject, status *XSetStatus) + UpdateScaleStrategy(ctx context.Context, c client.Client, object XSetObject, scaleStrategy *ScaleStrategy) error + GetXSetTemplatePatcher(object metav1.Object) func(client.Object) error +} +``` + +**Example (CollaSet):** +```go +func (s *XSetOperation) GetXSetSpec(object xsetapi.XSetObject) *xsetapi.XSetSpec { + set := object.(*CollaSet) + return &xsetapi.XSetSpec{ + Replicas: set.Spec.Replicas, + Paused: set.Spec.Paused, + Selector: set.Spec.Selector, + UpdateStrategy: convertUpdateStrategy(set.Spec.UpdateStrategy), + ScaleStrategy: convertScaleStrategy(set.Spec.ScaleStrategy), + HistoryLimit: set.Spec.HistoryLimit, + } +} +``` + +#### XOperation Interface + +```go +type XOperation interface { + GetXObjectFromRevision(revision *appsv1.ControllerRevision) (client.Object, error) + CheckScheduled(object client.Object) bool + CheckReadyTime(object client.Object) (bool, *metav1.Time) + CheckAvailable(object client.Object) bool + CheckInactive(object client.Object) bool + GetXOpsPriority(ctx context.Context, c client.Client, object client.Object) (*OpsPriority, error) +} +``` + +### 3. Implement Optional Interfaces + +#### ResourceContextAdapterGetter (for ID allocation) + +```go +func (r *ResourceContextAdapterGetter) GetResourceContextAdapter() xsetapi.ResourceContextAdapter { + return &MyResourceContextAdapter{} +} +``` + +#### LabelAnnotationManagerGetter (for custom labels) + +```go +func (g *LabelManagerAdapterGetter) GetLabelManagerAdapter() map[xsetapi.XSetLabelAnnotationEnum]string { + return map[xsetapi.XSetLabelAnnotationEnum]string{ + xsetapi.OperatingLabelPrefix: "my-operator/operating", + xsetapi.XInstanceIdLabelKey: "my-operator/instance-id", + // ... other labels + } +} +``` + +#### SubResourcePvcAdapter (for PVC management) + +```go +type SubResourcePvcAdapter interface { + RetainPvcWhenXSetDeleted(object XSetObject) bool + RetainPvcWhenXSetScaled(object XSetObject) bool + GetXSetPvcTemplate(object XSetObject) []corev1.PersistentVolumeClaim + GetXSpecVolumes(object client.Object) []corev1.Volume + GetXVolumeMounts(object client.Object) []corev1.VolumeMount + SetXSpecVolumes(object client.Object, volumes []corev1.Volume) +} +``` + +#### SubResourceAdapterGetter (for generic subresource management) + +Controllers implementing `SubResourcePvcAdapter` are automatically bridged to `SubResourceAdapter` via `BuildAdapters()`. For custom subresource types, implement `SubResourceAdapterGetter`: + +```go +func (c *MyXSetController) GetSubResourceAdapters() []xsetapi.SubResourceAdapter { + return []xsetapi.SubResourceAdapter{ + // Add custom adapters as needed + } +} +``` + +#### SubResourceAdapter Interface + +The `SubResourceAdapter` interface provides a generic way to manage subresources: + +```go +type SubResourceAdapter interface { + Meta() schema.GroupVersionKind + GetTemplates(xset XSetObject) ([]SubResourceTemplate, error) + RetainWhenXSetDeleted(xset XSetObject) bool + RetainWhenXSetScaled(xset XSetObject) bool + RecreateWhenXSetUpdated(xset XSetObject) bool + AttachToTarget(ctx context.Context, target client.Object, resources []client.Object) error +} + +// Optional interface for customizing resources +type SubResourceDecorator interface { + DecorateResource(ctx context.Context, xset XSetObject, template SubResourceTemplate, resource client.Object, target client.Object, targetID string) error +} +``` + +The control code creates resources from templates and sets namespace, owner reference, and labels. Implement `SubResourceDecorator` to customize the resource (e.g., set Name, add custom labels). + +#### Name Truncation + +Resource names are automatically truncated to 63 characters with a hash suffix for uniqueness: + +```go +truncator := subresources.NewNameTruncator() +name := truncator.Truncate("very-long-resource-name-exceeding-63-characters-limit") +// Result: "very-long-resource-name-exceeding-63-charact-abc123" +``` + +#### Label Value Handling + +Label values are automatically truncated with original value tracking: + +```go +lm := subresources.NewLabelManager(truncator) +lm.SetLabel(obj, "key", "very-long-label-value") +lm.SetLabelWithTrackedOriginal(obj, "key", "original-value-tracked-in-annotation") +``` + +#### DecorationAdapter (for decoration/patcher management) + +```go +type DecorationAdapter interface { + WatchDecoration(c controller.Controller) error + GetDecorationGroupVersionKind() metav1.GroupVersionKind + GetTargetCurrentDecorationRevisions(ctx context.Context, c client.Client, target client.Object) (string, error) + GetTargetUpdatedDecorationRevisions(ctx context.Context, c client.Client, target client.Object) (string, error) + GetDecorationPatcherByRevisions(ctx context.Context, c client.Client, target client.Object, revision string) (func(client.Object) error, error) + IsTargetDecorationChanged(currentRevision, updatedRevision string) (bool, error) +} +``` + +### 4. Register Controller + +Use `SetUpWithManager` to register your controller: + +```go +func Add(mgr ctrl.Manager) error { + xsetController := &MyXSetController{} + return xset.SetupWithManager(mgr, xsetController) +} +``` + +## Supported Features + +### Update Strategies + +| Strategy | Description | +|----------|-------------| +| `Recreate` | Delete and recreate targets on update | +| `InPlaceIfPossible` | In-place update if possible, fall back to recreate | +| `InPlaceOnly` | Always in-place update (requires special K8s cluster) | +| `Replace` | Create new target, wait for ready, then delete old | + +### Rolling Update Control + +- **ByPartition**: Control update progress by partition value +- **ByLabel**: Control update by attaching target labels + +### Scale Strategies + +- **Context Pool**: Share instance IDs between multiple XSets +- **TargetToInclude/Exclude**: Include/exclude specific targets +- **TargetToDelete**: Delete specific targets + +### OpsLifecycle + +Provides graceful operation lifecycle: +1. **Begin**: Start operation lifecycle +2. **AllowOps**: Check if operation is allowed (with delay) +3. **Finish**: Complete operation lifecycle + +Key functions in `opslifecycle/utils.go`: +- `Begin()` - Begin lifecycle +- `AllowOps()` - Check permission with delay +- `Finish()` - Finish lifecycle +- `IsDuringOps()` - Check if in lifecycle + +### ResourceContext + +Manages instance ID allocation across targets: +- `AllocateID()` - Allocate IDs for targets +- `CleanUnusedIDs()` - Clean unused IDs +- `UpdateToTargetContext()` - Update context + +## Label/Annotation Keys + +Key labels defined in `api/well_knowns.go`: + +| Label Type | Purpose | +|------------|---------| +| `OperatingLabelPrefix` | Target under operation | +| `OperationTypeLabelPrefix` | Type of operation | +| `OperateLabelPrefix` | Target can start operation | +| `XInstanceIdLabelKey` | Instance ID for target | +| `PreparingDeleteLabel` | Target preparing delete | +| `ControlledByXSetLabel` | Target controlled by XSet | + +## Example Implementations + +### CollaSet (kuperator) + +GitHub: https://github.com/KusionStack/kuperator + +Location: `pkg/controllers/collaset/` + +Key files: +- `collaset_controller.go` - XSetController implementation +- `collaset_adapter.go` - Adapters for XSetOperation, XOperation +- `resource_context.go` - ResourceContext adapter +- `lifecycle_adapter.go` - Lifecycle adapters + +CollaSet is the original implementation that kube-xset was extracted from. It manages Pod workloads with PodDecoration support for in-place updates. + +## Key Workflow + +The reconcile loop (in `xset_controller.go`): + +1. **SyncTargets**: Parse targets, allocate IDs, manage include/exclude +2. **Replace**: Handle replace-indicated targets +3. **Scale**: Scale out/in targets with lifecycle +4. **Update**: Update targets to new revision +5. **CalculateStatus**: Compute and update status + +## Testing + +Tests are located alongside source files (e.g., `*_test.go`). Key test patterns: + +- Use `suite` pattern for integration tests +- Mock clients for unit tests +- Focus on sync control logic + +## Import + +```go +import "kusionstack.io/kube-xset" +import xsetapi "kusionstack.io/kube-xset/api" +``` + +## Dependencies + +- `kusionstack.io/kube-utils` - Controller utilities +- `kusionstack.io/kube-api` - KusionStack API definitions +- Standard controller-runtime libraries \ No newline at end of file diff --git a/api/subresource_types.go b/api/subresource_types.go new file mode 100644 index 0000000..be49ba3 --- /dev/null +++ b/api/subresource_types.go @@ -0,0 +1,85 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package api + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// SubResourceAdapter is a generic interface for XSet subresources. +// Each subresource type (PVC, Service, ConfigMap, etc.) implements this interface. +type SubResourceAdapter interface { + // Meta returns the GroupVersionKind for this subresource type + Meta() schema.GroupVersionKind + + // GetTemplates returns subresource templates from XSet spec. + // The controller computes the Hash from each template using TemplateHash(). + GetTemplates(xset XSetObject) ([]SubResourceTemplate, error) + + // RetainWhenXSetDeleted returns true if subresource should be retained when XSet is deleted + RetainWhenXSetDeleted(xset XSetObject) bool + + // RetainWhenXSetScaled returns true if subresource should be retained when XSet is scaled in + RetainWhenXSetScaled(xset XSetObject) bool + + // RecreateWhenXSetUpdated returns true if subresource should be recreated + // when XSet is updating targets. When true, subresources are deleted and + // recreated during update operations regardless of template spec changes. + RecreateWhenXSetUpdated(xset XSetObject) bool + + // AttachToTarget attaches subresources to target (e.g., mount PVC volumes to Pod) + AttachToTarget(ctx context.Context, target client.Object, resources []client.Object) error + + // Optional interfaces: + // - SubResourceDecorator +} + +// SubResourceDecorator is an optional interface for customizing subresources. +// Implement this interface to customize the resource created from a template. +// +// IMPORTANT: Decorators MUST NOT modify the following fields as they are +// managed by the control code: +// - Namespace (set by control code to xset.Namespace) +// - OwnerReferences (set by control code with controller reference to xset) +// - Labels managed by control code: +// - ControlledByXSetLabel (or equivalent from XSetLabelAnnotationManager) +// - XInstanceIdLabelKey (set to targetID) +// - Template name label (e.g., SubResourcePvcTemplateLabelKey) +// - Template hash label (e.g., SubResourcePvcTemplateHashLabelKey) +// +// Decorators CAN: +// - Set Name (overrides GenerateName if set) +// - Add custom labels and annotations +// - Customize spec fields +// - Propagate additional labels from xset +type SubResourceDecorator interface { + DecorateResource(ctx context.Context, xset XSetObject, template SubResourceTemplate, resource, target client.Object, targetID string) error +} + +// SubResourceTemplate represents a parsed template with name and hash +type SubResourceTemplate struct { + // Name is the template name (e.g., "data", "logs") + Name string + // Hash is the hash of template spec for change detection. + // This is computed by the controller using TemplateHash(), users do not need to set it. + Hash string + // Template is the parsed template object + Template client.Object +} diff --git a/api/well_knowns.go b/api/well_knowns.go index a4cb6ed..86e8497 100644 --- a/api/well_knowns.go +++ b/api/well_knowns.go @@ -88,10 +88,20 @@ const ( // XExcludeIndicationLabelKey is used to indicate a target is excluded by xset XExcludeIndicationLabelKey - // SubResourcePvcTemplateLabelKey is used to attach pvc template name to pvc resources + // SubResourceTemplateLabelKey is used to attach template name to subresources. + // This is the generic name; SubResourcePvcTemplateLabelKey is deprecated but still works. + SubResourceTemplateLabelKey + + // SubResourceTemplateHashLabelKey is used to attach hash of template spec to subresources. + // This is the generic name; SubResourcePvcTemplateHashLabelKey is deprecated but still works. + SubResourceTemplateHashLabelKey + + // SubResourcePvcTemplateLabelKey is used to attach pvc template name to pvc resources. + // Deprecated: Use SubResourceTemplateLabelKey instead. SubResourcePvcTemplateLabelKey - // SubResourcePvcTemplateHashLabelKey is used to attach hash of pvc template to pvc subresource + // SubResourcePvcTemplateHashLabelKey is used to attach hash of pvc template to pvc subresource. + // Deprecated: Use SubResourceTemplateHashLabelKey instead. SubResourcePvcTemplateHashLabelKey // wellKnownCount is the number of XSetLabelAnnotationEnum @@ -125,6 +135,8 @@ var defaultXSetLabelAnnotationManager = map[XSetLabelAnnotationEnum]string{ XCreatingLabel: appsv1alpha1.PodCreatingLabel, XCompletingLabel: appsv1alpha1.PodCompletingLabel, XExcludeIndicationLabelKey: appsv1alpha1.PodExcludeIndicationLabelKey, + SubResourceTemplateLabelKey: appsv1alpha1.PvcTemplateLabelKey, + SubResourceTemplateHashLabelKey: appsv1alpha1.PvcTemplateHashLabelKey, SubResourcePvcTemplateLabelKey: appsv1alpha1.PvcTemplateLabelKey, SubResourcePvcTemplateHashLabelKey: appsv1alpha1.PvcTemplateHashLabelKey, } diff --git a/api/well_knowns_test.go b/api/well_knowns_test.go new file mode 100644 index 0000000..4d1832a --- /dev/null +++ b/api/well_knowns_test.go @@ -0,0 +1,53 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package api + +import ( + "testing" + + "github.com/onsi/gomega" +) + +func TestSubResourceTemplateLabelAliases(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + mgr := NewXSetLabelAnnotationManager(nil) + + // Test that new generic labels exist + templateKey := mgr.Value(SubResourceTemplateLabelKey) + templateHashKey := mgr.Value(SubResourceTemplateHashLabelKey) + + g.Expect(templateKey).ToNot(gomega.BeEmpty()) + g.Expect(templateHashKey).ToNot(gomega.BeEmpty()) +} + +func TestSubResourceTemplateLabelBackwardCompatibility(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + mgr := NewXSetLabelAnnotationManager(nil) + + // Test that PVC labels map to same values as generic labels + pvcTemplateKey := mgr.Value(SubResourcePvcTemplateLabelKey) + pvcTemplateHashKey := mgr.Value(SubResourcePvcTemplateHashLabelKey) + + templateKey := mgr.Value(SubResourceTemplateLabelKey) + templateHashKey := mgr.Value(SubResourceTemplateHashLabelKey) + + // Both should map to the same string values + g.Expect(pvcTemplateKey).To(gomega.Equal(templateKey)) + g.Expect(pvcTemplateHashKey).To(gomega.Equal(templateHashKey)) +} diff --git a/api/xset_controller_types.go b/api/xset_controller_types.go index d06ea95..235fba4 100644 --- a/api/xset_controller_types.go +++ b/api/xset_controller_types.go @@ -42,11 +42,13 @@ type XSetController interface { XOperation // Optional interfaces: - // - LifecycleAdapterGetter - // - ResourceContextAdapterGetter - // - LabelAnnotationManagerGetter - // - SubResourcePvcAdapter - // - DecorationAdapter + // - LifecycleAdapterGetter + // - ResourceContextAdapterGetter + // - LabelAnnotationManagerGetter + // - SubResourceAdapterGetter + // - SubResourcePvcAdapter + // - DecorationAdapter + // - TargetPrefixGetter } type XSetObject client.Object @@ -85,6 +87,12 @@ type LabelAnnotationManagerGetter interface { GetLabelManagerAdapter() map[XSetLabelAnnotationEnum]string } +// SubResourceAdapterGetter is used to get subresource adapters. +// Implement this to enable generic subresource management. +type SubResourceAdapterGetter interface { + GetSubResourceAdapters() []SubResourceAdapter +} + // SubResourcePvcAdapter is used to manage pvc subresource for X, which are declared on XSet, e.g., spec.volumeClaimTemplate. // Once adapter is implemented, XSetController will automatically manage pvc: (1) create pvcs from GetXSetPvcTemplate for each // X object and attach theses pvcs with same instance-id, (2) upgrade pvcs and recreate X object pvcs when PvcTemplateChanged, @@ -121,3 +129,11 @@ type DecorationAdapter interface { // IsTargetDecorationChanged returns true if decoration on target is changed. IsTargetDecorationChanged(currentRevision, updatedRevision string) (bool, error) } + +// TargetPrefixGetter is used to get custom prefix for target names. +// If not implemented or returns empty string, defaults to "{xset-name}-". +// Controller is responsible for truncation if needed. +// The returned prefix should end with "-" if a separator is desired. +type TargetPrefixGetter interface { + GetTargetPrefix(xset XSetObject) string +} diff --git a/docs/superpowers/specs/2026-03-30-generic-subresource-support-design.md b/docs/superpowers/specs/2026-03-30-generic-subresource-support-design.md new file mode 100644 index 0000000..8fcd6c9 --- /dev/null +++ b/docs/superpowers/specs/2026-03-30-generic-subresource-support-design.md @@ -0,0 +1,601 @@ +# kube-xset Generic SubResource Support Design + +**Date:** 2026-03-30 +**Author:** Claude +**Status:** Draft + +## Overview + +This design proposes adding generic subresource support to kube-xset, enabling XSetControllers to manage multiple subresource types (PVC, Service, etc.) beyond the current PVC-only support. + +### Goals + +1. **Generic SubResource Adapter** - Support multiple subresource types with a clean abstraction +2. **Name Truncation** - Automatically truncate resource names exceeding Kubernetes limits (63 chars for DNS labels) +3. **Label Value Handling** - Truncate label values with hash suffix for uniqueness +4. **Code Optimization** - Abstract PVC-specific code into reusable patterns +5. **Backward Compatibility** - Keep existing `SubResourcePvcAdapter` interface working + +### Non-Goals + +- Changing existing PVC behavior for controllers using `SubResourcePvcAdapter` +- Supporting stateful subresource ordering (that's ResourceContext's job) + +## Design + +### 1. Core Interfaces + +#### SubResourceAdapter Interface + +```go +// api/subresource_types.go + +package api + +import ( + "context" + "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// SubResourceAdapter is a generic interface for XSet subresources. +// Each subresource type (PVC, Service, ConfigMap, etc.) implements this interface. +type SubResourceAdapter interface { + // Meta returns the GroupVersionKind for this subresource type + Meta() schema.GroupVersionKind + + // GetTemplates returns subresource templates from XSet spec + GetTemplates(xset XSetObject) ([]SubResourceTemplate, error) + + // BuildResource creates a subresource instance from template for a specific target + BuildResource(ctx context.Context, xset XSetObject, template SubResourceTemplate, target client.Object, targetID string) (client.Object, error) + + // RetainWhenXSetDeleted returns true if subresource should be retained when XSet is deleted + RetainWhenXSetDeleted(xset XSetObject) bool + + // RetainWhenXSetScaled returns true if subresource should be retained when XSet is scaled in + RetainWhenXSetScaled(xset XSetObject) bool + + // AttachToTarget attaches subresources to target (e.g., mount PVC volumes to Pod) + AttachToTarget(ctx context.Context, target client.Object, resources []client.Object) error + + // GetAttachedResourceNames returns names of subresources attached to target + GetAttachedResourceNames(target client.Object) ([]string, error) +} + +// SubResourceTemplate represents a parsed template with name and hash +type SubResourceTemplate struct { + Name string // Template name (e.g., "data", "logs") + Hash string // Hash of template spec for change detection + Template client.Object // Parsed template object +} +``` + +#### SubResourceAdapterGetter Interface + +```go +// api/xset_controller_types.go - Add new optional interface + +// SubResourceAdapterGetter is used to get subresource adapters. +// Implement this to enable generic subresource management. +type SubResourceAdapterGetter interface { + GetSubResourceAdapters() []SubResourceAdapter +} +``` + +### 2. Name Truncation + +#### NameTruncator + +Uses Kubernetes standard constants from `k8s.io/apimachinery/pkg/util/validation`: + +```go +// subresources/types.go + +import ( + "fmt" + "hash/fnv" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/validation" +) + +// NameTruncator handles resource name truncation within Kubernetes limits +type NameTruncator struct { + MaxNameLength int +} + +func NewNameTruncator() *NameTruncator { + return &NameTruncator{ + MaxNameLength: validation.DNS1035LabelMaxLength, // 63 + } +} + +// Truncate truncates name if exceeds max, appending hash suffix for uniqueness +func (t *NameTruncator) Truncate(name string) string { + return t.TruncateWithMax(name, t.MaxNameLength) +} + +// TruncateWithMax truncates name to specific max length +func (t *NameTruncator) TruncateWithMax(name string, maxLen int) string { + if len(name) <= maxLen { + return name + } + + hash := computeHash(name) + hashSuffix := fmt.Sprintf("-%s", hash) + truncatedLen := maxLen - len(hashSuffix) + + if truncatedLen <= 0 { + return hashSuffix[1:] // remove leading dash + } + + return name[:truncatedLen] + hashSuffix +} + +// TruncateLabelValue truncates label value to 63 chars +func (t *NameTruncator) TruncateLabelValue(value string) string { + return t.TruncateWithMax(value, validation.LabelValueMaxLength) +} + +func computeHash(s string) string { + h := fnv.New32a() + h.Write([]byte(s)) + return rand.SafeEncodeString(fmt.Sprint(h.Sum32()))[:6] +} +``` + +### 3. Label Value Handling + +#### LabelManager + +```go +// subresources/types.go + +// LabelManager handles setting labels with automatic value truncation +type LabelManager struct { + truncator *NameTruncator +} + +func NewLabelManager(truncator *NameTruncator) *LabelManager { + return &LabelManager{ + truncator: truncator, + } +} + +// SetLabel sets a label, truncating value if needed with hash suffix +func (lm *LabelManager) SetLabel(obj client.Object, key, value string) { + if obj.GetLabels() == nil { + obj.SetLabels(make(map[string]string)) + } + + truncatedValue := lm.truncator.TruncateLabelValue(value) + obj.GetLabels()[key] = truncatedValue +} + +// SetLabelWithTrackedOriginal sets label and tracks original value in annotation +func (lm *LabelManager) SetLabelWithTrackedOriginal(obj client.Object, key, value string) { + truncatedValue := lm.truncator.TruncateLabelValue(value) + + if obj.GetLabels() == nil { + obj.SetLabels(make(map[string]string)) + } + obj.GetLabels()[key] = truncatedValue + + // Track original value in annotation if truncated + if truncatedValue != value { + if obj.GetAnnotations() == nil { + obj.SetAnnotations(make(map[string]string)) + } + obj.GetAnnotations()[key+".original"] = value + } +} + +// SetOperatingLabel sets operating label with ID in key +// Format: / = timestamp +func (lm *LabelManager) SetOperatingLabel(obj client.Object, prefix, id, value string) { + truncatedID := lm.truncator.TruncateLabelValue(id) + labelKey := fmt.Sprintf("%s/%s", prefix, truncatedID) + + if obj.GetLabels() == nil { + obj.SetLabels(make(map[string]string)) + } + obj.GetLabels()[labelKey] = value +} + +// SetRevisionLabel sets revision info as label value +func (lm *LabelManager) SetRevisionLabel(obj client.Object, prefix, id, revisionName string) { + truncatedID := lm.truncator.TruncateLabelValue(id) + truncatedRevision := lm.truncator.TruncateLabelValue(revisionName) + + labelKey := fmt.Sprintf("%s/%s", prefix, truncatedID) + + if obj.GetLabels() == nil { + obj.SetLabels(make(map[string]string)) + } + obj.GetLabels()[labelKey] = truncatedRevision + + if truncatedRevision != revisionName { + if obj.GetAnnotations() == nil { + obj.SetAnnotations(make(map[string]string)) + } + obj.GetAnnotations()[labelKey+".original-revision"] = revisionName + } +} +``` + +### 4. Generic SubResourceControl + +```go +// subresources/subresource_control.go + +// SubResourceControl manages lifecycle of all subresource types +type SubResourceControl interface { + // GetFilteredResources lists subresources owned by XSet + GetFilteredResources(ctx context.Context, xset api.XSetObject, gvk schema.GroupVersionKind) ([]client.Object, error) + + // CreateTargetResources creates subresources for a target + CreateTargetResources(ctx context.Context, xset api.XSetObject, target client.Object) error + + // DeleteTargetResources deletes subresources for a target + DeleteTargetResources(ctx context.Context, xset api.XSetObject, target client.Object) error + + // DeleteUnusedResources removes subresources no longer in templates + DeleteUnusedResources(ctx context.Context, xset api.XSetObject, target client.Object) error + + // AdoptOrphanedResources adopts resources left by retention policy + AdoptOrphanedResources(ctx context.Context, xset api.XSetObject) ([]client.Object, error) + + // OrphanResources removes owner reference (for retention) + OrphanResources(ctx context.Context, xset api.XSetObject, resources []client.Object) error + + // IsTemplateChanged checks if templates have changed + IsTemplateChanged(ctx context.Context, xset api.XSetObject, target client.Object) (bool, error) +} + +// RealSubResourceControl implements SubResourceControl using registered adapters +type RealSubResourceControl struct { + client client.Client + scheme *runtime.Scheme + adapters map[schema.GroupVersionKind]api.SubResourceAdapter + expectations *expectations.CacheExpectations + labelAnnoMgr api.XSetLabelAnnotationManager + xsetController api.XSetController + truncator *NameTruncator + labelManager *LabelManager +} + +// SubResourceControlBuilder builds control with registered adapters +type SubResourceControlBuilder struct { + adapters []api.SubResourceAdapter +} + +func NewSubResourceControlBuilder() *SubResourceControlBuilder { + return &SubResourceControlBuilder{} +} + +func (b *SubResourceControlBuilder) Register(adapter api.SubResourceAdapter) *SubResourceControlBuilder { + b.adapters = append(b.adapters, adapter) + return b +} + +func (b *SubResourceControlBuilder) Build( + mixin *mixin.ReconcilerMixin, + xsetController api.XSetController, + expectations *expectations.CacheExpectations, + labelAnnoMgr api.XSetLabelAnnotationManager, +) (SubResourceControl, error) { + adapters := make(map[schema.GroupVersionKind]api.SubResourceAdapter) + for _, adapter := range b.adapters { + adapters[adapter.Meta()] = adapter + } + + truncator := NewNameTruncator() + + return &RealSubResourceControl{ + client: mixin.Client, + scheme: mixin.Scheme, + adapters: adapters, + expectations: expectations, + labelAnnoMgr: labelAnnoMgr, + xsetController: xsetController, + truncator: truncator, + labelManager: NewLabelManager(truncator), + }, nil +} +``` + +### 5. PVC Adapter Implementation + +```go +// subresources/pvc_adapter.go + +// PvcSubResourceAdapter implements SubResourceAdapter for PVC +// Also implements old SubResourcePvcAdapter for backward compatibility +type PvcSubResourceAdapter struct { + xsetController api.XSetController + labelAnnoMgr api.XSetLabelAnnotationManager + truncator *NameTruncator + labelManager *LabelManager +} + +func NewPvcSubResourceAdapter(xsetController api.XSetController, labelAnnoMgr api.XSetLabelAnnotationManager) *PvcSubResourceAdapter { + truncator := NewNameTruncator() + return &PvcSubResourceAdapter{ + xsetController: xsetController, + labelAnnoMgr: labelAnnoMgr, + truncator: truncator, + labelManager: NewLabelManager(truncator), + } +} + +func (p *PvcSubResourceAdapter) Meta() schema.GroupVersionKind { + return corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim") +} + +func (p *PvcSubResourceAdapter) GetTemplates(xset api.XSetObject) ([]api.SubResourceTemplate, error) { + // Use old interface for backward compatibility if XSetController implements it + if pvcAdapter, ok := p.xsetController.(api.SubResourcePvcAdapter); ok { + templates := pvcAdapter.GetXSetPvcTemplate(xset) + var result []api.SubResourceTemplate + for i := range templates { + hash, err := PvcTemplateHash(&templates[i]) + if err != nil { + return nil, err + } + result = append(result, api.SubResourceTemplate{ + Name: templates[i].Name, + Hash: hash, + Template: &templates[i], + }) + } + return result, nil + } + return nil, nil +} + +func (p *PvcSubResourceAdapter) BuildResource( + ctx context.Context, + xset api.XSetObject, + template api.SubResourceTemplate, + target client.Object, + targetID string, +) (client.Object, error) { + pvc := template.Template.(*corev1.PersistentVolumeClaim).DeepCopy() + + // Generate name: xsetname-templatename-targetid + baseName := fmt.Sprintf("%s-%s-%s", xset.GetName(), template.Name, targetID) + pvc.Name = p.truncator.Truncate(baseName) + pvc.Namespace = xset.GetNamespace() + + // Set owner reference + xsetMeta := xset.GetObjectKind().GroupVersionKind() + pvc.OwnerReferences = []metav1.OwnerReference{ + *metav1.NewControllerRef(xset, xsetMeta), + } + + // Set labels + if pvc.Labels == nil { + pvc.Labels = make(map[string]string) + } + p.labelManager.SetLabel(pvc, p.labelAnnoMgr.Value(api.ControlledByXSetLabel), "true") + p.labelManager.SetLabel(pvc, p.labelAnnoMgr.Value(api.XInstanceIdLabelKey), targetID) + p.labelManager.SetLabelWithTrackedOriginal(pvc, p.labelAnnoMgr.Value(api.SubResourcePvcTemplateLabelKey), template.Name) + p.labelManager.SetLabel(pvc, p.labelAnnoMgr.Value(api.SubResourcePvcTemplateHashLabelKey), template.Hash) + + return pvc, nil +} + +func (p *PvcSubResourceAdapter) AttachToTarget(ctx context.Context, target client.Object, resources []client.Object) error { + // Build volumes from PVCs and set on target + var volumes []corev1.Volume + for _, res := range resources { + pvc := res.(*corev1.PersistentVolumeClaim) + templateName := pvc.Labels[p.labelAnnoMgr.Value(api.SubResourcePvcTemplateLabelKey)] + volumes = append(volumes, corev1.Volume{ + Name: templateName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.Name, + }, + }, + }) + } + + // Use old interface to set volumes on target + if pvcAdapter, ok := p.xsetController.(api.SubResourcePvcAdapter); ok { + pvcAdapter.SetXSpecVolumes(target, volumes) + } + return nil +} + +func (p *PvcSubResourceAdapter) RetainWhenXSetDeleted(xset api.XSetObject) bool { + if pvcAdapter, ok := p.xsetController.(api.SubResourcePvcAdapter); ok { + return pvcAdapter.RetainPvcWhenXSetDeleted(xset) + } + return false +} + +func (p *PvcSubResourceAdapter) RetainWhenXSetScaled(xset api.XSetObject) bool { + if pvcAdapter, ok := p.xsetController.(api.SubResourcePvcAdapter); ok { + return pvcAdapter.RetainPvcWhenXSetScaled(xset) + } + return false +} + +func (p *PvcSubResourceAdapter) GetAttachedResourceNames(target client.Object) ([]string, error) { + if pvcAdapter, ok := p.xsetController.(api.SubResourcePvcAdapter); ok { + volumes := pvcAdapter.GetXSpecVolumes(target) + var names []string + for _, v := range volumes { + if v.PersistentVolumeClaim != nil { + names = append(names, v.PersistentVolumeClaim.ClaimName) + } + } + return names, nil + } + return nil, nil +} +``` + +### 6. Service Adapter Example + +```go +// subresources/service_adapter.go + +// ServiceSubResourceAdapter implements SubResourceAdapter for Service +type ServiceSubResourceAdapter struct { + truncator *NameTruncator + labelManager *LabelManager +} + +func NewServiceSubResourceAdapter() *ServiceSubResourceAdapter { + truncator := NewNameTruncator() + return &ServiceSubResourceAdapter{ + truncator: truncator, + labelManager: NewLabelManager(truncator), + } +} + +func (s *ServiceSubResourceAdapter) Meta() schema.GroupVersionKind { + return corev1.SchemeGroupVersion.WithKind("Service") +} + +func (s *ServiceSubResourceAdapter) GetTemplates(xset api.XSetObject) ([]api.SubResourceTemplate, error) { + // Get from XSet spec or annotation + // Example: annotation with JSON service templates + // Or new field in XSetSpec + return nil, nil // implement based on XSet type +} + +func (s *ServiceSubResourceAdapter) BuildResource( + ctx context.Context, + xset api.XSetObject, + template api.SubResourceTemplate, + target client.Object, + targetID string, +) (client.Object, error) { + svc := template.Template.(*corev1.Service).DeepCopy() + + baseName := fmt.Sprintf("%s-%s-%s", xset.GetName(), template.Name, targetID) + svc.Name = s.truncator.Truncate(baseName) + svc.Namespace = xset.GetNamespace() + + // Set owner reference + xsetMeta := xset.GetObjectKind().GroupVersionKind() + svc.OwnerReferences = []metav1.OwnerReference{ + *metav1.NewControllerRef(xset, xsetMeta), + } + + // Set labels for selection + if svc.Labels == nil { + svc.Labels = make(map[string]string) + } + s.labelManager.SetLabel(svc, "app.kubernetes.io/instance", targetID) + + return svc, nil +} + +// Service doesn't need to "attach" to target in the same way PVC does +func (s *ServiceSubResourceAdapter) AttachToTarget(ctx context.Context, target client.Object, resources []client.Object) error { + return nil // Services are independent, no attachment needed +} + +func (s *ServiceSubResourceAdapter) GetAttachedResourceNames(target client.Object) ([]string, error) { + return nil, nil // Services are independent, no attachment to target +} + +func (s *ServiceSubResourceAdapter) RetainWhenXSetDeleted(xset api.XSetObject) bool { + // Services are typically deleted with XSet by default + return false +} + +func (s *ServiceSubResourceAdapter) RetainWhenXSetScaled(xset api.XSetObject) bool { + return false +} +``` + +## File Structure + +``` +kube-xset/ +├── api/ +│ ├── xset_controller_types.go # Add SubResourceAdapterGetter interface +│ ├── subresource_types.go # NEW: SubResourceAdapter, SubResourceTemplate interfaces +│ └── ... (other existing files) +├── subresources/ +│ ├── subresource_control.go # NEW: Generic SubResourceControl +│ ├── subresource_control_test.go # NEW +│ ├── types.go # NEW: NameTruncator, LabelManager +│ ├── utils.go # NEW: Shared utilities (hash, etc.) +│ ├── getter.go # UPDATED: Add GetSubResourceAdapter() +│ ├── pvc_control.go # KEPT for backward compatibility +│ ├── pvc_adapter.go # NEW: PvcSubResourceAdapter +│ └── service_adapter.go # NEW: ServiceSubResourceAdapter (example) +└── ... (other existing files) +``` + +## Migration Plan + +### Phase 1: Add New Interfaces and Utilities (No Breaking Changes) + +1. Add `api/subresource_types.go` with new interfaces +2. Add `subresources/types.go` with `NameTruncator`, `LabelManager` +3. Add `subresources/utils.go` with shared utilities +4. Update `subresources/getter.go` to add `GetSubResourceAdapter()` function + +### Phase 2: Implement PVC Adapter with New Interface + +1. Add `subresources/pvc_adapter.go` implementing both interfaces +2. Add `subresources/subresource_control.go` generic control +3. Update existing `pvc_control.go` to use new utilities internally + +### Phase 3: Integration + +1. Update `xset_controller.go` to use new `SubResourceControl` +2. Keep old `SubResourcePvcAdapter` path for backward compatibility +3. Prefer new `SubResourceAdapterGetter` if implemented + +### Phase 4: Example Implementation + +1. Add `subresources/service_adapter.go` as reference implementation +2. Update CLAUDE.md with new subresource documentation + +## Backward Compatibility + +| XSetController Implements | Behavior | +|--------------------------|----------| +| Neither interface | No subresource management | +| `SubResourcePvcAdapter` only | Uses old `PvcControl` (unchanged) | +| `SubResourceAdapterGetter` only | Uses new `SubResourceControl` | +| Both | Prefers new `SubResourceAdapterGetter` | + +## Testing Strategy + +1. **Unit Tests** + - `NameTruncator.Truncate()` with various name lengths + - `LabelManager.SetLabel()` with long values + - Hash uniqueness for truncated names + +2. **Integration Tests** + - PVC adapter with new interface + - Service adapter example + - Backward compatibility with old `SubResourcePvcAdapter` + +3. **E2E Tests** + - Create XSet with long names, verify truncation + - Update templates, verify subresource recreation + - Delete XSet, verify retention policy + +## Open Questions + +1. Should we add a `ValidateTemplates()` method to `SubResourceAdapter` for admission validation? +2. Should `NameTruncator` be configurable per resource type (e.g., 253 for ConfigMaps)? +3. Should we track original names in annotations for debugging purposes? + +## References + +- Kubernetes naming constraints: `k8s.io/apimachinery/pkg/util/validation` +- Existing PVC implementation: `subresources/pvc_control.go` +- Example implementations: + - ModelSet: `/Users/ana/projects/aicloud/modelops-controller/pkg/controllers/modelset/` + - LeaderSet: `/Users/ana/projects/aicloud/aether/pkg/controller/leaderworkerset/leaderset/` \ No newline at end of file diff --git a/subresources/getter.go b/subresources/getter.go index 7ff5d2c..bfc9d85 100644 --- a/subresources/getter.go +++ b/subresources/getter.go @@ -16,9 +16,166 @@ package subresources -import "kusionstack.io/kube-xset/api" +import ( + "context" + "fmt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kube-xset/api" +) + +// PVCGvk is the GroupVersionKind for PersistentVolumeClaim. +var PVCGvk = corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim") + +// GetSubresourcePvcAdapter returns the PVC adapter if the controller implements SubResourcePvcAdapter. func GetSubresourcePvcAdapter(control api.XSetController) (adapter api.SubResourcePvcAdapter, enabled bool) { adapter, enabled = control.(api.SubResourcePvcAdapter) return adapter, enabled } + +// GetSubResourceAdapters returns subresource adapters if the controller implements SubResourceAdapterGetter. +func GetSubResourceAdapters(control api.XSetController) (adapters []api.SubResourceAdapter, enabled bool) { + getter, ok := control.(api.SubResourceAdapterGetter) + if !ok { + return nil, false + } + return getter.GetSubResourceAdapters(), true +} + +// BuildAdapters builds the adapter list with auto-bridge for legacy controllers. +// Priority: +// 1. If controller implements SubResourceAdapterGetter, use its adapters +// 2. Else if controller implements SubResourcePvcAdapter, auto-bridge to SubResourceControl +// 3. Else return nil (no subresource management) +func BuildAdapters(controller api.XSetController, labelAnnoMgr api.XSetLabelAnnotationManager) []api.SubResourceAdapter { + // Priority 1: Controller provides its own adapters + if getter, ok := controller.(api.SubResourceAdapterGetter); ok { + return getter.GetSubResourceAdapters() + } + + // Priority 2: Auto-bridge legacy SubResourcePvcAdapter + if _, ok := controller.(api.SubResourcePvcAdapter); ok { + return []api.SubResourceAdapter{ + NewPvcSubResourceAdapter(controller, labelAnnoMgr), + } + } + + // No subresource management + return nil +} + +// PvcSubResourceAdapter implements SubResourceAdapter for PVC. +// It bridges to the legacy SubResourcePvcAdapter for backward compatibility. +type PvcSubResourceAdapter struct { + xsetController api.XSetController + labelAnnoMgr api.XSetLabelAnnotationManager +} + +// NewPvcSubResourceAdapter creates a new PVC adapter. +func NewPvcSubResourceAdapter(xsetController api.XSetController, labelAnnoMgr api.XSetLabelAnnotationManager) *PvcSubResourceAdapter { + return &PvcSubResourceAdapter{ + xsetController: xsetController, + labelAnnoMgr: labelAnnoMgr, + } +} + +// Meta returns the GVK for PVC. +func (p *PvcSubResourceAdapter) Meta() schema.GroupVersionKind { + return PVCGvk +} + +// GetTemplates returns PVC templates from the XSet. +func (p *PvcSubResourceAdapter) GetTemplates(xset api.XSetObject) ([]api.SubResourceTemplate, error) { + pvcAdapter, ok := p.xsetController.(api.SubResourcePvcAdapter) + if !ok { + return nil, nil + } + + templates := pvcAdapter.GetXSetPvcTemplate(xset) + var result []api.SubResourceTemplate + for i := range templates { + hash, err := TemplateHash(&templates[i]) + if err != nil { + return nil, fmt.Errorf("failed to compute PVC template hash: %w", err) + } + result = append(result, api.SubResourceTemplate{ + Name: templates[i].Name, + Hash: hash, + Template: &templates[i], + }) + } + return result, nil +} + +// RetainWhenXSetDeleted returns whether PVCs should be retained when XSet is deleted. +func (p *PvcSubResourceAdapter) RetainWhenXSetDeleted(xset api.XSetObject) bool { + if pvcAdapter, ok := p.xsetController.(api.SubResourcePvcAdapter); ok { + return pvcAdapter.RetainPvcWhenXSetDeleted(xset) + } + return false +} + +// RetainWhenXSetScaled returns whether PVCs should be retained when XSet is scaled in. +func (p *PvcSubResourceAdapter) RetainWhenXSetScaled(xset api.XSetObject) bool { + if pvcAdapter, ok := p.xsetController.(api.SubResourcePvcAdapter); ok { + return pvcAdapter.RetainPvcWhenXSetScaled(xset) + } + return false +} + +// RecreateWhenXSetUpdated returns false by default for backward compatibility. +// PVCs are recreated only when template spec changes (hash mismatch). +func (p *PvcSubResourceAdapter) RecreateWhenXSetUpdated(xset api.XSetObject) bool { + return false +} + +// AttachToTarget attaches PVCs to the target by setting volumes. +func (p *PvcSubResourceAdapter) AttachToTarget(ctx context.Context, target client.Object, resources []client.Object) error { + if len(resources) == 0 { + return nil + } + + pvcAdapter, ok := p.xsetController.(api.SubResourcePvcAdapter) + if !ok { + return nil + } + + var volumes []corev1.Volume + for _, res := range resources { + pvc, ok := res.(*corev1.PersistentVolumeClaim) + if !ok { + continue + } + templateName := pvc.Labels[p.labelAnnoMgr.Value(api.SubResourceTemplateLabelKey)] + volumes = append(volumes, corev1.Volume{ + Name: templateName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.Name, + }, + }, + }) + } + + existingVolumes := pvcAdapter.GetXSpecVolumes(target) + volumeMap := make(map[string]corev1.Volume, len(existingVolumes)+len(volumes)) + for i := range existingVolumes { + volumeMap[existingVolumes[i].Name] = existingVolumes[i] + } + for i := range volumes { + volumeMap[volumes[i].Name] = volumes[i] + } + + mergedVolumes := make([]corev1.Volume, 0, len(volumeMap)) + for _, v := range volumeMap { //nolint:gocritic // unavoidable when building slice from map values + mergedVolumes = append(mergedVolumes, v) + } + + pvcAdapter.SetXSpecVolumes(target, mergedVolumes) + return nil +} + +var _ api.SubResourceAdapter = &PvcSubResourceAdapter{} diff --git a/subresources/getter_test.go b/subresources/getter_test.go new file mode 100644 index 0000000..4245b9f --- /dev/null +++ b/subresources/getter_test.go @@ -0,0 +1,243 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package subresources + +import ( + "context" + "testing" + + "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kube-xset/api" +) + +// mockControllerWithoutAdapters is a controller that implements neither interface +type mockControllerWithoutAdapters struct{} + +func (m *mockControllerWithoutAdapters) ControllerName() string { return "mock" } +func (m *mockControllerWithoutAdapters) FinalizerName() string { return "mock/finalizer" } +func (m *mockControllerWithoutAdapters) XSetMeta() metav1.TypeMeta { + return metav1.TypeMeta{Kind: "MockSet", APIVersion: "v1"} +} + +func (m *mockControllerWithoutAdapters) XMeta() metav1.TypeMeta { + return metav1.TypeMeta{Kind: "Mock", APIVersion: "v1"} +} +func (m *mockControllerWithoutAdapters) NewXSetObject() api.XSetObject { return nil } +func (m *mockControllerWithoutAdapters) NewXObject() client.Object { return nil } +func (m *mockControllerWithoutAdapters) NewXObjectList() client.ObjectList { + return &corev1.PodList{} +} + +func (m *mockControllerWithoutAdapters) GetXSetSpec(object api.XSetObject) *api.XSetSpec { + return nil +} + +func (m *mockControllerWithoutAdapters) GetXSetPatch(object metav1.Object) ([]byte, error) { + return nil, nil +} + +func (m *mockControllerWithoutAdapters) GetXSetStatus(object api.XSetObject) *api.XSetStatus { + return nil +} + +func (m *mockControllerWithoutAdapters) SetXSetStatus(object api.XSetObject, status *api.XSetStatus) { +} + +func (m *mockControllerWithoutAdapters) UpdateScaleStrategy(ctx context.Context, c client.Client, object api.XSetObject, scaleStrategy *api.ScaleStrategy) error { + return nil +} + +func (m *mockControllerWithoutAdapters) GetXSetTemplatePatcher(object metav1.Object) func(client.Object) error { + return nil +} + +func (m *mockControllerWithoutAdapters) GetXObjectFromRevision(revision *appsv1.ControllerRevision) (client.Object, error) { + return nil, nil +} +func (m *mockControllerWithoutAdapters) CheckScheduled(object client.Object) bool { return false } +func (m *mockControllerWithoutAdapters) CheckReadyTime(object client.Object) (bool, *metav1.Time) { + return false, nil +} +func (m *mockControllerWithoutAdapters) CheckAvailable(object client.Object) bool { return false } +func (m *mockControllerWithoutAdapters) CheckInactive(object client.Object) bool { return false } +func (m *mockControllerWithoutAdapters) GetXOpsPriority(ctx context.Context, c client.Client, object client.Object) (*api.OpsPriority, error) { + return nil, nil +} + +func (m *mockControllerWithoutAdapters) GetTargetPrefix(xset api.XSetObject) string { + return "" +} + +// mockControllerWithPvcAdapter implements SubResourcePvcAdapter +type mockControllerWithPvcAdapter struct { + mockControllerWithoutAdapters +} + +func (m *mockControllerWithPvcAdapter) RetainPvcWhenXSetDeleted(object api.XSetObject) bool { + return true +} + +func (m *mockControllerWithPvcAdapter) RetainPvcWhenXSetScaled(object api.XSetObject) bool { + return false +} + +func (m *mockControllerWithPvcAdapter) GetXSetPvcTemplate(object api.XSetObject) []corev1.PersistentVolumeClaim { + return nil +} + +func (m *mockControllerWithPvcAdapter) GetXSpecVolumes(object client.Object) []corev1.Volume { + return nil +} + +func (m *mockControllerWithPvcAdapter) GetXVolumeMounts(object client.Object) []corev1.VolumeMount { + return nil +} + +func (m *mockControllerWithPvcAdapter) SetXSpecVolumes(object client.Object, pvcs []corev1.Volume) { +} + +// mockSubResourceAdapter is a mock adapter for testing +type mockSubResourceAdapter struct{} + +func (m *mockSubResourceAdapter) Meta() schema.GroupVersionKind { + return schema.GroupVersionKind{Group: "test", Version: "v1", Kind: "MockResource"} +} + +func (m *mockSubResourceAdapter) GetTemplates(xset api.XSetObject) ([]api.SubResourceTemplate, error) { + return nil, nil +} +func (m *mockSubResourceAdapter) RetainWhenXSetDeleted(xset api.XSetObject) bool { return false } +func (m *mockSubResourceAdapter) RetainWhenXSetScaled(xset api.XSetObject) bool { return false } +func (m *mockSubResourceAdapter) RecreateWhenXSetUpdated(xset api.XSetObject) bool { return false } +func (m *mockSubResourceAdapter) AttachToTarget(ctx context.Context, target client.Object, resources []client.Object) error { + return nil +} + +// mockControllerWithAdapterGetter implements SubResourceAdapterGetter +type mockControllerWithAdapterGetter struct { + mockControllerWithoutAdapters + adapters []api.SubResourceAdapter +} + +func (m *mockControllerWithAdapterGetter) GetSubResourceAdapters() []api.SubResourceAdapter { + return m.adapters +} + +// mockControllerWithBothInterfaces implements both SubResourceAdapterGetter and SubResourcePvcAdapter +type mockControllerWithBothInterfaces struct { + mockControllerWithPvcAdapter + adapters []api.SubResourceAdapter +} + +func (m *mockControllerWithBothInterfaces) GetSubResourceAdapters() []api.SubResourceAdapter { + return m.adapters +} + +func TestBuildAdapters_NoAdapters(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + controller := &mockControllerWithoutAdapters{} + labelAnnoMgr := api.NewXSetLabelAnnotationManager(nil) + + result := BuildAdapters(controller, labelAnnoMgr) + + g.Expect(result).To(gomega.BeNil()) +} + +func TestBuildAdapters_PvcAdapterAutoBridge(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + controller := &mockControllerWithPvcAdapter{} + labelAnnoMgr := api.NewXSetLabelAnnotationManager(nil) + + result := BuildAdapters(controller, labelAnnoMgr) + + g.Expect(result).ToNot(gomega.BeNil()) + g.Expect(result).To(gomega.HaveLen(1)) + g.Expect(result[0]).To(gomega.BeAssignableToTypeOf(&PvcSubResourceAdapter{})) +} + +func TestBuildAdapters_AdapterGetter(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + mockAdapter := &mockSubResourceAdapter{} + controller := &mockControllerWithAdapterGetter{ + adapters: []api.SubResourceAdapter{mockAdapter}, + } + labelAnnoMgr := api.NewXSetLabelAnnotationManager(nil) + + result := BuildAdapters(controller, labelAnnoMgr) + + g.Expect(result).ToNot(gomega.BeNil()) + g.Expect(result).To(gomega.HaveLen(1)) + g.Expect(result[0]).To(gomega.Equal(mockAdapter)) +} + +func TestBuildAdapters_AdapterGetterMultipleAdapters(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + mockAdapter1 := &mockSubResourceAdapter{} + mockAdapter2 := &mockSubResourceAdapter{} + controller := &mockControllerWithAdapterGetter{ + adapters: []api.SubResourceAdapter{mockAdapter1, mockAdapter2}, + } + labelAnnoMgr := api.NewXSetLabelAnnotationManager(nil) + + result := BuildAdapters(controller, labelAnnoMgr) + + g.Expect(result).ToNot(gomega.BeNil()) + g.Expect(result).To(gomega.HaveLen(2)) +} + +func TestBuildAdapters_AdapterGetterPriorityOverPvc(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + // Controller implements both interfaces - AdapterGetter should take priority + mockAdapter := &mockSubResourceAdapter{} + controller := &mockControllerWithBothInterfaces{ + adapters: []api.SubResourceAdapter{mockAdapter}, + } + labelAnnoMgr := api.NewXSetLabelAnnotationManager(nil) + + result := BuildAdapters(controller, labelAnnoMgr) + + g.Expect(result).ToNot(gomega.BeNil()) + g.Expect(result).To(gomega.HaveLen(1)) + // Should use the custom adapter, not auto-bridged PVC adapter + g.Expect(result[0]).To(gomega.Equal(mockAdapter)) +} + +func TestBuildAdapters_EmptyAdapterList(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + controller := &mockControllerWithAdapterGetter{ + adapters: []api.SubResourceAdapter{}, + } + labelAnnoMgr := api.NewXSetLabelAnnotationManager(nil) + + result := BuildAdapters(controller, labelAnnoMgr) + + // Empty slice from getter should be returned as-is + g.Expect(result).ToNot(gomega.BeNil()) + g.Expect(result).To(gomega.BeEmpty()) +} diff --git a/subresources/pvc_control.go b/subresources/pvc_control.go deleted file mode 100644 index 7c38d7a..0000000 --- a/subresources/pvc_control.go +++ /dev/null @@ -1,554 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package subresources - -import ( - "context" - "encoding/json" - "fmt" - "hash/fnv" - "strings" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/rand" - "k8s.io/apimachinery/pkg/util/sets" - kubeutilclient "kusionstack.io/kube-utils/client" - "kusionstack.io/kube-utils/controller/expectations" - "kusionstack.io/kube-utils/controller/mixin" - refmanagerutil "kusionstack.io/kube-utils/controller/refmanager" - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" - - "kusionstack.io/kube-xset/api" -) - -const ( - FieldIndexOwnerRefUID = "ownerRefUID" -) - -var PVCGvk = corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim") - -type PvcControl interface { - GetFilteredPvcs(context.Context, api.XSetObject) ([]*corev1.PersistentVolumeClaim, error) - CreateTargetPvcs(context.Context, api.XSetObject, client.Object, []*corev1.PersistentVolumeClaim) error - DeleteTargetPvcs(context.Context, api.XSetObject, client.Object, []*corev1.PersistentVolumeClaim) error - DeleteTargetUnusedPvcs(context.Context, api.XSetObject, client.Object, []*corev1.PersistentVolumeClaim) error - OrphanPvc(context.Context, api.XSetObject, *corev1.PersistentVolumeClaim) error - AdoptPvc(context.Context, api.XSetObject, *corev1.PersistentVolumeClaim) error - AdoptPvcsLeftByRetainPolicy(context.Context, api.XSetObject) ([]*corev1.PersistentVolumeClaim, error) - IsTargetPvcTmpChanged(api.XSetObject, client.Object, []*corev1.PersistentVolumeClaim) (bool, error) - RetainPvcWhenXSetDeleted(xset api.XSetObject) bool - RetainPvcWhenXSetScaled(xset api.XSetObject) bool -} - -type RealPvcControl struct { - client client.Client - scheme *runtime.Scheme - pvcAdapter api.SubResourcePvcAdapter - expectations *expectations.CacheExpectations - xsetLabelAnnoMgr api.XSetLabelAnnotationManager - xsetController api.XSetController -} - -func NewRealPvcControl(mixin *mixin.ReconcilerMixin, expectations *expectations.CacheExpectations, xsetLabelAnnoMgr api.XSetLabelAnnotationManager, xsetController api.XSetController) (PvcControl, error) { - // requires implementation of SubResourcePvcAdapter - pvcAdapter, ok := GetSubresourcePvcAdapter(xsetController) - if !ok { - return nil, nil - } - // here we go, set up cache and return real pvc control - if err := setUpCache(mixin.Cache, xsetController); err != nil { - return nil, err - } - return &RealPvcControl{ - client: mixin.Client, - scheme: mixin.Scheme, - pvcAdapter: pvcAdapter, - expectations: expectations, - xsetLabelAnnoMgr: xsetLabelAnnoMgr, - xsetController: xsetController, - }, nil -} - -func (pc *RealPvcControl) GetFilteredPvcs(ctx context.Context, xset api.XSetObject) ([]*corev1.PersistentVolumeClaim, error) { - // list pvcs using ownerReference - var filteredPvcs []*corev1.PersistentVolumeClaim - ownedPvcList := &corev1.PersistentVolumeClaimList{} - if err := pc.client.List(ctx, ownedPvcList, &client.ListOptions{ - Namespace: xset.GetNamespace(), - FieldSelector: fields.OneTermEqualSelector(FieldIndexOwnerRefUID, string(xset.GetUID())), - }); err != nil { - return nil, err - } - - for i := range ownedPvcList.Items { - pvc := &ownedPvcList.Items[i] - if pvc.DeletionTimestamp == nil { - filteredPvcs = append(filteredPvcs, pvc) - } - } - return filteredPvcs, nil -} - -func (pc *RealPvcControl) CreateTargetPvcs(ctx context.Context, xset api.XSetObject, x client.Object, existingPvcs []*corev1.PersistentVolumeClaim) error { - id, exist := pc.xsetLabelAnnoMgr.Get(x, api.XInstanceIdLabelKey) - if !exist { - return nil - } - - // provision pvcs related to pod using pvc template, and reuse - // pvcs if "instance-id" and "pvc-template-hash" label matched - pvcsMap, err := pc.provisionUpdatedPvc(ctx, id, xset, existingPvcs) - if err != nil { - return err - } - - newVolumes := make([]corev1.Volume, 0, len(pvcsMap)) - // mount updated pvcs to target - for name, pvc := range pvcsMap { - volume := corev1.Volume{ - Name: name, - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc.Name, - ReadOnly: false, - }, - }, - } - newVolumes = append(newVolumes, volume) - } - - // append legacy pvcs - currentVolumes := pc.pvcAdapter.GetXSpecVolumes(x) - for i := range currentVolumes { - currentVolume := currentVolumes[i] - if _, ok := pvcsMap[currentVolume.Name]; !ok { - newVolumes = append(newVolumes, currentVolume) - } - } - pc.pvcAdapter.SetXSpecVolumes(x, newVolumes) - return nil -} - -func (pc *RealPvcControl) provisionUpdatedPvc(ctx context.Context, id string, xset api.XSetObject, existingPvcs []*corev1.PersistentVolumeClaim) (map[string]*corev1.PersistentVolumeClaim, error) { - updatedPvcs, _, err := pc.classifyTargetPvcs(id, xset, existingPvcs) - if err != nil { - return nil, err - } - - templates := pc.pvcAdapter.GetXSetPvcTemplate(xset) - for i := range templates { - pvcTmp := templates[i] - // reuse pvc - if _, exist := updatedPvcs[pvcTmp.Name]; exist { - continue - } - // create new pvc - claim, err := pc.buildPvcWithHash(id, xset, &pvcTmp) - if err != nil { - return nil, err - } - - if err := pc.client.Create(ctx, claim); err != nil { - return nil, fmt.Errorf("fail to create pvc for id %s: %w", id, err) - } - - if err := pc.expectations.ExpectCreation( - kubeutilclient.ObjectKeyString(xset), - PVCGvk, - claim.Namespace, - claim.Name, - ); err != nil { - return nil, err - } - - updatedPvcs[pvcTmp.Name] = claim - } - return updatedPvcs, nil -} - -func (pc *RealPvcControl) DeleteTargetPvcs(ctx context.Context, xset api.XSetObject, x client.Object, pvcs []*corev1.PersistentVolumeClaim) error { - for _, pvc := range pvcs { - if pvc.Labels == nil || x.GetLabels() == nil { - continue - } - - // only delete pvcs used by target - pvcId, _ := pc.xsetLabelAnnoMgr.Get(pvc, api.XInstanceIdLabelKey) - targetId, _ := pc.xsetLabelAnnoMgr.Get(x, api.XInstanceIdLabelKey) - if pvcId != targetId { - continue - } - - if err := deletePvcWithExpectations(ctx, pc.client, xset, pc.expectations, pvc); err != nil { - return err - } - } - return nil -} - -func (pc *RealPvcControl) DeleteTargetUnusedPvcs(ctx context.Context, xset api.XSetObject, x client.Object, existingPvcs []*corev1.PersistentVolumeClaim) error { - id, exist := pc.xsetLabelAnnoMgr.Get(x, api.XInstanceIdLabelKey) - if !exist { - return nil - } - - newPvcs, oldPvcs, err := pc.classifyTargetPvcs(id, xset, existingPvcs) - if err != nil { - return err - } - - volumeMounts := pc.pvcAdapter.GetXVolumeMounts(x) - mountedVolumeTmps := sets.String{} - for i := range volumeMounts { - mountedVolumeTmps.Insert(volumeMounts[i].Name) - } - - // delete pvc which is not claimed in templates - if err := pc.deleteUnclaimedPvcs(ctx, xset, oldPvcs, mountedVolumeTmps); err != nil { - return err - } - // delete old pvc if new pvc is provisioned and not RetainPVCWhenXSetScaled - if !pc.pvcAdapter.RetainPvcWhenXSetScaled(xset) { - return pc.deleteOldPvcs(ctx, xset, newPvcs, oldPvcs) - } - return nil -} - -func (pc *RealPvcControl) AdoptPvc(ctx context.Context, xset api.XSetObject, pvc *corev1.PersistentVolumeClaim) error { - xsetSpec := pc.xsetController.GetXSetSpec(xset) - if xsetSpec.Selector.MatchLabels == nil { - return nil - } - refWriter := refmanagerutil.NewOwnerRefWriter(pc.client) - matcher, err := refmanagerutil.LabelSelectorAsMatch(xsetSpec.Selector) - if err != nil { - return fmt.Errorf("fail to create labelSelector matcher: %w", err) - } - refManager := refmanagerutil.NewObjectControllerRefManager(refWriter, xset, xset.GetObjectKind().GroupVersionKind(), matcher) - - if _, err := refManager.Claim(ctx, pvc); err != nil { - return fmt.Errorf("failed to adopt pvc: %w", err) - } - return nil -} - -func (pc *RealPvcControl) OrphanPvc(ctx context.Context, xset api.XSetObject, pvc *corev1.PersistentVolumeClaim) error { - xsetSpec := pc.xsetController.GetXSetSpec(xset) - if xsetSpec.Selector.MatchLabels == nil { - return nil - } - if pvc.Labels == nil { - pvc.Labels = make(map[string]string) - } - if pvc.Annotations == nil { - pvc.Annotations = make(map[string]string) - } - - refWriter := refmanagerutil.NewOwnerRefWriter(pc.client) - if err := refWriter.Release(ctx, xset, pvc); err != nil { - return fmt.Errorf("failed to orphan target: %w", err) - } - return nil -} - -func (pc *RealPvcControl) AdoptPvcsLeftByRetainPolicy(ctx context.Context, xset api.XSetObject) ([]*corev1.PersistentVolumeClaim, error) { - xsetSpec := pc.xsetController.GetXSetSpec(xset) - ownerSelector := xsetSpec.Selector.DeepCopy() - if ownerSelector.MatchLabels == nil { - ownerSelector.MatchLabels = map[string]string{} - } - ownerSelector.MatchLabels[pc.xsetLabelAnnoMgr.Value(api.ControlledByXSetLabel)] = "true" - ownerSelector.MatchExpressions = append(ownerSelector.MatchExpressions, metav1.LabelSelectorRequirement{ // nolint - Key: pc.xsetLabelAnnoMgr.Value(api.XOrphanedIndicationLabelKey), // should not be excluded pvcs - Operator: metav1.LabelSelectorOpDoesNotExist, - }) - ownerSelector.MatchExpressions = append(ownerSelector.MatchExpressions, metav1.LabelSelectorRequirement{ - Key: pc.xsetLabelAnnoMgr.Value(api.XInstanceIdLabelKey), // instance-id label should exist - Operator: metav1.LabelSelectorOpExists, - }) - ownerSelector.MatchExpressions = append(ownerSelector.MatchExpressions, metav1.LabelSelectorRequirement{ - Key: pc.xsetLabelAnnoMgr.Value(api.SubResourcePvcTemplateHashLabelKey), // pvc-hash label should exist - Operator: metav1.LabelSelectorOpExists, - }) - - selector, err := metav1.LabelSelectorAsSelector(ownerSelector) - if err != nil { - return nil, err - } - - orphanedPvcList := &corev1.PersistentVolumeClaimList{} - if err := pc.client.List(ctx, orphanedPvcList, &client.ListOptions{Namespace: xset.GetNamespace(), LabelSelector: selector}); err != nil { - return nil, err - } - - // adopt orphaned pvcs - var claims []*corev1.PersistentVolumeClaim - for i := range orphanedPvcList.Items { - pvc := orphanedPvcList.Items[i] - if pvc.OwnerReferences != nil && len(pvc.OwnerReferences) > 0 { - continue - } - if pvc.Labels == nil { - pvc.Labels = make(map[string]string) - } - if pvc.Annotations == nil { - pvc.Annotations = make(map[string]string) - } - - claims = append(claims, &pvc) - } - for i := range claims { - if err := pc.AdoptPvc(ctx, xset, claims[i]); err != nil { - return nil, err - } - } - return claims, nil -} - -func (pc *RealPvcControl) IsTargetPvcTmpChanged(xset api.XSetObject, x client.Object, existingPvcs []*corev1.PersistentVolumeClaim) (bool, error) { - pvcTemplates := pc.pvcAdapter.GetXSetPvcTemplate(xset) - xSpecVolumes := pc.pvcAdapter.GetXSpecVolumes(x) - // get pvc template hash values - newHashMapping, err := PvcTmpHashMapping(pvcTemplates) - if err != nil { - return false, err - } - - // get existing x pvcs hash values - existingPvcHash := map[string]string{} - for _, pvc := range existingPvcs { - if pvc.Labels == nil || x.GetLabels() == nil { - continue - } - pvcId, _ := pc.xsetLabelAnnoMgr.Get(pvc, api.XInstanceIdLabelKey) - targetId, _ := pc.xsetLabelAnnoMgr.Get(x, api.XInstanceIdLabelKey) - if pvcId != targetId { - continue - } - if v, exist := pc.xsetLabelAnnoMgr.Get(pvc, api.SubResourcePvcTemplateHashLabelKey); !exist { - continue - } else { - existingPvcHash[pvc.Name] = v - } - } - - // check mounted pvcs changed - for i := range xSpecVolumes { - volume := xSpecVolumes[i] - if volume.PersistentVolumeClaim == nil || volume.PersistentVolumeClaim.ClaimName == "" { - continue - } - pvcName := volume.PersistentVolumeClaim.ClaimName - TmpName := volume.Name - if newHashMapping[TmpName] != existingPvcHash[pvcName] { - return true, nil - } - } - return false, nil -} - -func (pc *RealPvcControl) RetainPvcWhenXSetDeleted(xset api.XSetObject) bool { - return pc.pvcAdapter.RetainPvcWhenXSetDeleted(xset) -} - -func (pc *RealPvcControl) RetainPvcWhenXSetScaled(xset api.XSetObject) bool { - return pc.pvcAdapter.RetainPvcWhenXSetScaled(xset) -} - -func (pc *RealPvcControl) deleteUnclaimedPvcs(ctx context.Context, xset api.XSetObject, oldPvcs map[string]*corev1.PersistentVolumeClaim, mountedPvcNames sets.String) error { - inUsedPvcNames := sets.String{} - templates := pc.pvcAdapter.GetXSetPvcTemplate(xset) - for i := range templates { - inUsedPvcNames.Insert(templates[i].Name) - } - for pvcTmpName, pvc := range oldPvcs { - // if pvc is still mounted on target, keep it - if mountedPvcNames.Has(pvcTmpName) { - continue - } - - // is pvc is claimed in pvc templates, keep it - if inUsedPvcNames.Has(pvcTmpName) { - continue - } - - if err := deletePvcWithExpectations(ctx, pc.client, xset, pc.expectations, pvc); err != nil { - return err - } - } - return nil -} - -func (pc *RealPvcControl) deleteOldPvcs(ctx context.Context, xset api.XSetObject, newPvcs, oldPvcs map[string]*corev1.PersistentVolumeClaim) error { - for pvcTmpName, pvc := range oldPvcs { - if _, newPvcExist := newPvcs[pvcTmpName]; !newPvcExist { - continue - } - if err := deletePvcWithExpectations(ctx, pc.client, xset, pc.expectations, pvc); err != nil { - return err - } - } - return nil -} - -func (pc *RealPvcControl) buildPvcWithHash(id string, xset api.XSetObject, pvcTmp *corev1.PersistentVolumeClaim) (*corev1.PersistentVolumeClaim, error) { - claim := pvcTmp.DeepCopy() - claim.Name = "" - claim.GenerateName = fmt.Sprintf("%s-%s-", xset.GetName(), pvcTmp.Name) - claim.Namespace = xset.GetNamespace() - xsetMeta := pc.xsetController.XSetMeta() - xsetGvk := xsetMeta.GroupVersionKind() - claim.OwnerReferences = append(claim.OwnerReferences, - *metav1.NewControllerRef(xset, xsetGvk)) - - if claim.Labels == nil { - claim.Labels = map[string]string{} - } - xsetSpec := pc.xsetController.GetXSetSpec(xset) - for k, v := range xsetSpec.Selector.MatchLabels { - claim.Labels[k] = v - } - pc.xsetLabelAnnoMgr.Set(claim, api.ControlledByXSetLabel, "true") - - hash, err := PvcTmpHash(pvcTmp) - if err != nil { - return nil, err - } - pc.xsetLabelAnnoMgr.Set(claim, api.SubResourcePvcTemplateHashLabelKey, hash) - pc.xsetLabelAnnoMgr.Set(claim, api.XInstanceIdLabelKey, id) - pc.xsetLabelAnnoMgr.Set(claim, api.SubResourcePvcTemplateLabelKey, pvcTmp.Name) - return claim, nil -} - -// classify pvcs into old and new ones -func (pc *RealPvcControl) classifyTargetPvcs(id string, xset api.XSetObject, existingPvcs []*corev1.PersistentVolumeClaim) (map[string]*corev1.PersistentVolumeClaim, map[string]*corev1.PersistentVolumeClaim, error) { - newPvcs := map[string]*corev1.PersistentVolumeClaim{} - oldPvcs := map[string]*corev1.PersistentVolumeClaim{} - - newPvcTemplates := pc.pvcAdapter.GetXSetPvcTemplate(xset) - newTmpHash, err := PvcTmpHashMapping(newPvcTemplates) - if err != nil { - return newPvcs, oldPvcs, err - } - - for _, pvc := range existingPvcs { - if pvc.DeletionTimestamp != nil { - continue - } - - if pvc.Labels == nil { - continue - } - - if val, exist := pc.xsetLabelAnnoMgr.Get(pvc, api.XInstanceIdLabelKey); !exist { - continue - } else if val != id { - continue - } - - if _, exist := pc.xsetLabelAnnoMgr.Get(pvc, api.SubResourcePvcTemplateHashLabelKey); !exist { - continue - } - hash, _ := pc.xsetLabelAnnoMgr.Get(pvc, api.SubResourcePvcTemplateHashLabelKey) - pvcTmpName, err := pc.extractPvcTmpName(xset, pvc) - if err != nil { - return nil, nil, err - } - - // classify into updated and old pvcs - if newTmpHash[pvcTmpName] == hash { - newPvcs[pvcTmpName] = pvc - } else { - oldPvcs[pvcTmpName] = pvc - } - } - - return newPvcs, oldPvcs, nil -} - -func (pc *RealPvcControl) extractPvcTmpName(xset api.XSetObject, pvc *corev1.PersistentVolumeClaim) (string, error) { - if pvcTmpName, exist := pc.xsetLabelAnnoMgr.Get(pvc, api.SubResourcePvcTemplateLabelKey); exist { - return pvcTmpName, nil - } - lastDashIndex := strings.LastIndex(pvc.Name, "-") - if lastDashIndex == -1 { - return "", fmt.Errorf("pvc %s has no postfix", pvc.Name) - } - - rest := pvc.Name[:lastDashIndex] - if !strings.HasPrefix(rest, xset.GetName()+"-") { - return "", fmt.Errorf("malformed pvc name %s, expected a part of CollaSet name %s", pvc.Name, xset.GetName()) - } - - return strings.TrimPrefix(rest, xset.GetName()+"-"), nil -} - -func PvcTmpHash(pvc *corev1.PersistentVolumeClaim) (string, error) { - bytes, err := json.Marshal(pvc) - if err != nil { - return "", fmt.Errorf("fail to marshal pvc template: %w", err) - } - - hf := fnv.New32() - if _, err = hf.Write(bytes); err != nil { - return "", fmt.Errorf("fail to calculate pvc template hash: %w", err) - } - - return rand.SafeEncodeString(fmt.Sprint(hf.Sum32())), nil -} - -func PvcTmpHashMapping(pvcTmps []corev1.PersistentVolumeClaim) (map[string]string, error) { - pvcHashMapping := map[string]string{} - for i := range pvcTmps { - pvcTmp := pvcTmps[i] - hash, err := PvcTmpHash(&pvcTmp) - if err != nil { - return nil, err - } - pvcHashMapping[pvcTmp.Name] = hash - } - return pvcHashMapping, nil -} - -func deletePvcWithExpectations(ctx context.Context, client client.Client, xset api.XSetObject, expectations *expectations.CacheExpectations, pvc *corev1.PersistentVolumeClaim) error { - if err := client.Delete(ctx, pvc); err != nil { - return err - } - - // expect deletion - if err := expectations.ExpectDeletion(kubeutilclient.ObjectKeyString(xset), PVCGvk, pvc.GetNamespace(), pvc.GetName()); err != nil { - return err - } - return nil -} - -func setUpCache(cache cache.Cache, controller api.XSetController) error { - if err := cache.IndexField(context.TODO(), &corev1.PersistentVolumeClaim{}, FieldIndexOwnerRefUID, func(object client.Object) []string { - ownerRef := metav1.GetControllerOf(object) - if ownerRef == nil || ownerRef.Kind != controller.XSetMeta().Kind { - return nil - } - return []string{string(ownerRef.UID)} - }); err != nil { - return fmt.Errorf("failed to index by field for pvc->xset %s: %w", FieldIndexOwnerRefUID, err) - } - return nil -} diff --git a/subresources/subresource_control.go b/subresources/subresource_control.go new file mode 100644 index 0000000..898f1b2 --- /dev/null +++ b/subresources/subresource_control.go @@ -0,0 +1,955 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package subresources + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + kubeutilclient "kusionstack.io/kube-utils/client" + "kusionstack.io/kube-utils/controller/expectations" + "kusionstack.io/kube-utils/controller/mixin" + refmanagerutil "kusionstack.io/kube-utils/controller/refmanager" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kube-xset/api" +) + +// FieldIndexOwnerRefUID is the field index for owner reference UID. +const FieldIndexOwnerRefUID = "ownerRefUID" + +// SubResourceState wraps a subresource with its adapter metadata. +type SubResourceState struct { + // Object is the actual subresource (PVC, Service, etc.) + Object client.Object + // GVK is the GroupVersionKind of the subresource + GVK schema.GroupVersionKind + // Adapter is the adapter that manages this subresource type + Adapter api.SubResourceAdapter +} + +// SubResourceControl manages all subresource types through registered adapters. +type SubResourceControl interface { + // Lifecycle operations (called from sync_control.go) + + // GetFilteredResources lists all subresources owned by the XSet + GetFilteredResources(ctx context.Context, xset api.XSetObject) ([]SubResourceState, error) + + // AdoptOrphanedResources adopts subresources left by retention policy + AdoptOrphanedResources(ctx context.Context, xset api.XSetObject) ([]SubResourceState, error) + + // CreateTargetResources creates subresources for a target and attaches them + CreateTargetResources(ctx context.Context, xset api.XSetObject, target client.Object, existing []SubResourceState) error + + // DeleteTargetResources deletes subresources for a target. + // If isReplaceTarget is true, deletes all resources. Otherwise, only deletes resources + // where the adapter's RetainWhenXSetScaled returns false. + DeleteTargetResources(ctx context.Context, xset api.XSetObject, target client.Object, existing []SubResourceState, isReplaceTarget bool) error + + // DeleteTargetUnusedResources deletes unused subresources for a target + DeleteTargetUnusedResources(ctx context.Context, xset api.XSetObject, target client.Object, existing []SubResourceState) error + + // DeleteTargetRecreateResources deletes subresources for a target that need recreation on update. + // Only deletes resources for adapters where RecreateWhenXSetUpdated returns true. + DeleteTargetRecreateResources(ctx context.Context, xset api.XSetObject, target client.Object, existing []SubResourceState) error + + // OrphanResource removes owner reference from a subresource + OrphanResource(ctx context.Context, xset api.XSetObject, resource client.Object) error + + // Query operations + + // IsTargetTemplateChanged returns true if any subresource template changed for target, + // or if the target has resources that need recreation on update (when isUpdatedRevision is false). + IsTargetTemplateChanged(xset api.XSetObject, target client.Object, existing []SubResourceState, isUpdatedRevision bool) (bool, error) + + // ReclaimSubResourcesOnDeletion handles subresource retention when XSet is being deleted. + // It fetches all subresources and orphans those marked for retention. + ReclaimSubResourcesOnDeletion(ctx context.Context, xset api.XSetObject) error + + // Include/Exclude support + + // CheckAllowIncludeExclude checks if target's subresources allow include/exclude + CheckAllowIncludeExclude(ctx context.Context, xset api.XSetObject, target client.Object, fn CheckAllowFunc) (bool, error) + + // AdoptTargetResources adopts subresources for a target during include operation + AdoptTargetResources(ctx context.Context, xset api.XSetObject, target client.Object, instanceID string) error + + // OrphanTargetResources orphans all subresources for a target during exclude operation + OrphanTargetResources(ctx context.Context, xset api.XSetObject, target client.Object) error + + // AdoptSingleResource adopts a single subresource by setting owner reference. + AdoptSingleResource(ctx context.Context, xset api.XSetObject, resource client.Object) error +} + +// CheckAllowFunc is the function type for checking include/exclude permission. +// Defined in synccontrols package: func(obj client.Object, ownerName, ownerKind string, labelMgr api.XSetLabelAnnotationManager) (bool, string) +type CheckAllowFunc func(obj client.Object, ownerName, ownerKind string, labelMgr api.XSetLabelAnnotationManager) (bool, string) + +// RealSubResourceControl implements SubResourceControl with multiple adapters. +type RealSubResourceControl struct { + client client.Client + scheme *runtime.Scheme + adaptersByGVK map[schema.GroupVersionKind]api.SubResourceAdapter + expectations *expectations.CacheExpectations + labelAnnoMgr api.XSetLabelAnnotationManager + xsetController api.XSetController +} + +// NewRealSubResourceControl creates a new SubResourceControl. +// Returns nil if no adapters are provided (subresource management disabled). +func NewRealSubResourceControl( + mixin *mixin.ReconcilerMixin, + adapters []api.SubResourceAdapter, + expectations *expectations.CacheExpectations, + labelAnnoMgr api.XSetLabelAnnotationManager, + xsetController api.XSetController, +) (SubResourceControl, error) { + if len(adapters) == 0 { + return nil, nil + } + + // Build GVK index from adapters + adaptersByGVK := make(map[schema.GroupVersionKind]api.SubResourceAdapter) + for _, adapter := range adapters { + gvk := adapter.Meta() + adaptersByGVK[gvk] = adapter + } + + // Set up cache indexes for all adapter GVKs + if err := setUpCacheForAdapters(mixin.Cache, mixin.Scheme, adapters, xsetController); err != nil { + return nil, err + } + + return &RealSubResourceControl{ + client: mixin.Client, + scheme: mixin.Scheme, + adaptersByGVK: adaptersByGVK, + expectations: expectations, + labelAnnoMgr: labelAnnoMgr, + xsetController: xsetController, + }, nil +} + +// setUpCacheForAdapters registers field indexes for all adapter GVKs. +func setUpCacheForAdapters(cache cache.Cache, scheme *runtime.Scheme, adapters []api.SubResourceAdapter, controller api.XSetController) error { + for _, adapter := range adapters { + gvk := adapter.Meta() + obj, err := scheme.New(gvk) + if err != nil { + return fmt.Errorf("failed to set up cache for adapter GVK %s: type is not registered in the scheme; ensure this type is added to the controller scheme: %w", gvk, err) + } + if err := cache.IndexField(context.TODO(), obj.(client.Object), FieldIndexOwnerRefUID, func(object client.Object) []string { + ownerRef := metav1.GetControllerOf(object) + if ownerRef == nil || ownerRef.Kind != controller.XSetMeta().Kind { + return nil + } + return []string{string(ownerRef.UID)} + }); err != nil { + return fmt.Errorf("failed to index by field for %s->xset %s: %w", gvk.Kind, FieldIndexOwnerRefUID, err) + } + } + return nil +} + +// newListForGVK creates a new list object for the given GVK using the scheme. +func (sc *RealSubResourceControl) newListForGVK(gvk schema.GroupVersionKind) (client.ObjectList, error) { + listGVK := gvk.GroupVersion().WithKind(gvk.Kind + "List") + obj, err := sc.scheme.New(listGVK) + if err != nil { + return nil, fmt.Errorf("failed to create list for GVK %s: %w", gvk, err) + } + return obj.(client.ObjectList), nil +} + +// newObjectForGVK creates a new object for the given GVK using the scheme. +func (sc *RealSubResourceControl) newObjectForGVK(gvk schema.GroupVersionKind) (client.Object, error) { + obj, err := sc.scheme.New(gvk) + if err != nil { + return nil, fmt.Errorf("failed to create object for GVK %s: %w", gvk, err) + } + return obj.(client.Object), nil +} + +// GetFilteredResources lists all subresources owned by the XSet. +func (sc *RealSubResourceControl) GetFilteredResources(ctx context.Context, xset api.XSetObject) ([]SubResourceState, error) { + var resources []SubResourceState + + for _, adapter := range sc.adaptersByGVK { + gvk := adapter.Meta() + list, err := sc.newListForGVK(gvk) + if err != nil { + continue + } + + if err := sc.client.List(ctx, list, &client.ListOptions{ + Namespace: xset.GetNamespace(), + FieldSelector: fields.OneTermEqualSelector(FieldIndexOwnerRefUID, string(xset.GetUID())), + }); err != nil { + return nil, fmt.Errorf("failed to list %s: %w", gvk.Kind, err) + } + + items := extractListItems(list) + for _, item := range items { + if item.GetDeletionTimestamp() == nil { + resources = append(resources, SubResourceState{ + Object: item, + GVK: gvk, + Adapter: adapter, + }) + } + } + } + + return resources, nil +} + +// extractListItems extracts items from any ObjectList using reflection. +func extractListItems(list client.ObjectList) []client.Object { + items := make([]client.Object, 0) + if err := meta.EachListItem(list, func(obj runtime.Object) error { + items = append(items, obj.(client.Object)) + return nil + }); err != nil { + return nil + } + return items +} + +// AdoptOrphanedResources adopts subresources left by retention policy. +func (sc *RealSubResourceControl) AdoptOrphanedResources(ctx context.Context, xset api.XSetObject) ([]SubResourceState, error) { + var adopted []SubResourceState + + for _, adapter := range sc.adaptersByGVK { + if adapter.RetainWhenXSetDeleted(xset) { + // Find orphaned resources for this adapter + orphaned, err := sc.findOrphanedResources(ctx, xset, adapter) + if err != nil { + return nil, err + } + + for _, res := range orphaned { + if err := sc.adoptResource(ctx, xset, res); err != nil { + return nil, err + } + adopted = append(adopted, SubResourceState{ + Object: res, + GVK: adapter.Meta(), + Adapter: adapter, + }) + } + } + } + + return adopted, nil +} + +// findOrphanedResources finds subresources that have the controlled-by label but no owner reference. +func (sc *RealSubResourceControl) findOrphanedResources(ctx context.Context, xset api.XSetObject, adapter api.SubResourceAdapter) ([]client.Object, error) { + xsetSpec := sc.xsetController.GetXSetSpec(xset) + ownerSelector := xsetSpec.Selector.DeepCopy() + if ownerSelector.MatchLabels == nil { + ownerSelector.MatchLabels = map[string]string{} + } + ownerSelector.MatchLabels[sc.labelAnnoMgr.Value(api.ControlledByXSetLabel)] = "true" + ownerSelector.MatchExpressions = append(ownerSelector.MatchExpressions, + metav1.LabelSelectorRequirement{ + Key: sc.labelAnnoMgr.Value(api.XOrphanedIndicationLabelKey), + Operator: metav1.LabelSelectorOpDoesNotExist, + }, + metav1.LabelSelectorRequirement{ + Key: sc.labelAnnoMgr.Value(api.XInstanceIdLabelKey), + Operator: metav1.LabelSelectorOpExists, + }, + ) + + selector, err := metav1.LabelSelectorAsSelector(ownerSelector) + if err != nil { + return nil, err + } + + gvk := adapter.Meta() + list, err := sc.newListForGVK(gvk) + if err != nil { + return nil, nil + } + + if err := sc.client.List(ctx, list, &client.ListOptions{ + Namespace: xset.GetNamespace(), + LabelSelector: selector, + }); err != nil { + return nil, err + } + + items := extractListItems(list) + var orphaned []client.Object + for _, item := range items { + if len(item.GetOwnerReferences()) == 0 { + orphaned = append(orphaned, item) + } + } + return orphaned, nil +} + +// adoptResource sets the owner reference on a subresource. +func (sc *RealSubResourceControl) adoptResource(ctx context.Context, xset api.XSetObject, res client.Object) error { + xsetSpec := sc.xsetController.GetXSetSpec(xset) + if xsetSpec.Selector.MatchLabels == nil { + return nil + } + + refWriter := refmanagerutil.NewOwnerRefWriter(sc.client) + matcher, err := refmanagerutil.LabelSelectorAsMatch(xsetSpec.Selector) + if err != nil { + return fmt.Errorf("fail to create labelSelector matcher: %w", err) + } + refManager := refmanagerutil.NewObjectControllerRefManager(refWriter, xset, xset.GetObjectKind().GroupVersionKind(), matcher) + + if _, err := refManager.Claim(ctx, res); err != nil { + return fmt.Errorf("failed to adopt subresource: %w", err) + } + return nil +} + +// AdoptSingleResource adopts a single subresource by setting owner reference. +func (sc *RealSubResourceControl) AdoptSingleResource(ctx context.Context, xset api.XSetObject, resource client.Object) error { + return sc.adoptResource(ctx, xset, resource) +} + +// classifyResourcesByHash classifies resources into new and old based on template hash. +// Returns two maps: newResources (hash matches current template) and oldResources (hash differs). +func (sc *RealSubResourceControl) classifyResourcesByHash(targetID string, xset api.XSetObject, adapter api.SubResourceAdapter, existing []SubResourceState) (map[string]SubResourceState, map[string]SubResourceState, error) { + newResources := make(map[string]SubResourceState) + oldResources := make(map[string]SubResourceState) + + gvk := adapter.Meta() + + // Get current template hashes + templates, err := adapter.GetTemplates(xset) + if err != nil { + return newResources, oldResources, err + } + templateHashes := make(map[string]string) + for _, tmpl := range templates { + templateHashes[tmpl.Name] = tmpl.Hash + } + + // Classify existing resources for this adapter + for _, state := range existing { + if state.GVK != gvk { + continue + } + + // Skip resources being deleted + if state.Object.GetDeletionTimestamp() != nil { + continue + } + + // Only process resources for this target + resourceID, exist := sc.labelAnnoMgr.Get(state.Object, api.XInstanceIdLabelKey) + if !exist || resourceID != targetID { + continue + } + + // Get template hash and name + resourceHash, exist := sc.labelAnnoMgr.Get(state.Object, api.SubResourceTemplateHashLabelKey) + if !exist { + continue + } + + templateName, _ := sc.labelAnnoMgr.Get(state.Object, api.SubResourceTemplateLabelKey) + + // Classify by hash comparison + if currentHash, ok := templateHashes[templateName]; ok && currentHash == resourceHash { + newResources[templateName] = state + } else { + oldResources[templateName] = state + } + } + + return newResources, oldResources, nil +} + +// filterByTarget returns resources belonging to the given target. +func (sc *RealSubResourceControl) filterByTarget(existing []SubResourceState, target client.Object) []SubResourceState { + targetID, _ := sc.labelAnnoMgr.Get(target, api.XInstanceIdLabelKey) + if targetID == "" { + return nil + } + var result []SubResourceState + for _, state := range existing { + if resourceID, _ := sc.labelAnnoMgr.Get(state.Object, api.XInstanceIdLabelKey); resourceID == targetID { + result = append(result, state) + } + } + return result +} + +// hasRecreateOnUpdateResources returns true if target has resources that need recreation on update. +func (sc *RealSubResourceControl) hasRecreateOnUpdateResources(xset api.XSetObject, target client.Object, existing []SubResourceState) bool { + for _, state := range sc.filterByTarget(existing, target) { + if state.Adapter != nil && state.Adapter.RecreateWhenXSetUpdated(xset) { + return true + } + } + return false +} + +// orphanRetainedResources orphans resources marked for retention on XSet deletion. +func (sc *RealSubResourceControl) orphanRetainedResources(ctx context.Context, xset api.XSetObject, existing []SubResourceState) error { + for _, state := range existing { + if state.Adapter != nil && state.Adapter.RetainWhenXSetDeleted(xset) && len(state.Object.GetOwnerReferences()) > 0 { + if err := sc.OrphanResource(ctx, xset, state.Object); err != nil { + return err + } + } + } + return nil +} + +// ReclaimSubResourcesOnDeletion handles subresource retention when XSet is being deleted. +// It fetches all subresources and orphans those marked for retention. +func (sc *RealSubResourceControl) ReclaimSubResourcesOnDeletion(ctx context.Context, xset api.XSetObject) error { + resources, err := sc.GetFilteredResources(ctx, xset) + if err != nil { + return err + } + return sc.orphanRetainedResources(ctx, xset, resources) +} + +// deleteResource deletes a subresource and tracks the expectation. +// It issues a normal delete and lets Kubernetes handle finalizers naturally. +// Resources with finalizers (e.g., PVCs with kubernetes.io/pvc-protection) will +// get a deletion timestamp and be deleted when their finalizers are cleared. +func (sc *RealSubResourceControl) deleteResource(ctx context.Context, xset api.XSetObject, resource client.Object, gvk schema.GroupVersionKind) error { + if err := sc.client.Delete(ctx, resource); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete %s %s: %w", gvk.Kind, resource.GetName(), err) + } + + // Track expectation for deletion + if err := sc.expectations.ExpectDeletion( + kubeutilclient.ObjectKeyString(xset), + gvk, + resource.GetNamespace(), + resource.GetName(), + ); err != nil { + return err + } + + return nil +} + +// CreateTargetResources creates subresources for a target and attaches them. +func (sc *RealSubResourceControl) CreateTargetResources(ctx context.Context, xset api.XSetObject, target client.Object, existing []SubResourceState) error { + id, exist := sc.labelAnnoMgr.Get(target, api.XInstanceIdLabelKey) + if !exist { + return nil + } + + for _, adapter := range sc.adaptersByGVK { + if err := sc.createResourcesForAdapter(ctx, xset, target, existing, id, adapter); err != nil { + return err + } + } + + return nil +} + +// createResourcesForAdapter creates subresources for a specific adapter. +func (sc *RealSubResourceControl) createResourcesForAdapter(ctx context.Context, xset api.XSetObject, target client.Object, existing []SubResourceState, targetID string, adapter api.SubResourceAdapter) error { + // Get desired templates + templates, err := adapter.GetTemplates(xset) + if err != nil { + return fmt.Errorf("failed to get templates from adapter %s: %w", adapter.Meta().Kind, err) + } + if len(templates) == 0 { + return nil + } + + // Classify existing resources for this adapter and target + gvk := adapter.Meta() + existingByTemplateName := make(map[string]SubResourceState) + for _, state := range existing { + if state.GVK == gvk { + // Only consider resources belonging to this target + resourceID, _ := sc.labelAnnoMgr.Get(state.Object, api.XInstanceIdLabelKey) + if resourceID != targetID { + continue + } + templateName, _ := sc.labelAnnoMgr.Get(state.Object, api.SubResourceTemplateLabelKey) + if templateName != "" { + existingByTemplateName[templateName] = state + } + } + } + + // Create resources and build list for attachment + var createdResources []client.Object + for _, template := range templates { + // Check if we can reuse existing resource + if existing, ok := existingByTemplateName[template.Name]; ok { + existingHash, _ := sc.labelAnnoMgr.Get(existing.Object, api.SubResourceTemplateHashLabelKey) + if existingHash == template.Hash { + // Reuse existing — hash matches, no need to recreate + createdResources = append(createdResources, existing.Object) + continue + } + // Delete old resource before creating new one (hash changed) + if err := sc.deleteResource(ctx, xset, existing.Object, gvk); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete old %s %s: %w", gvk.Kind, existing.Object.GetName(), err) + } + } + + // Create new resource from template + resource := template.Template.DeepCopyObject().(client.Object) + + // Set namespace + resource.SetNamespace(xset.GetNamespace()) + + // Set owner reference + xsetMeta := sc.xsetController.XSetMeta() + resource.SetOwnerReferences([]metav1.OwnerReference{ + *metav1.NewControllerRef(xset, xsetMeta.GroupVersionKind()), + }) + + // Set labels + labels := resource.GetLabels() + if labels == nil { + labels = make(map[string]string) + } + labels[sc.labelAnnoMgr.Value(api.ControlledByXSetLabel)] = "true" + labels[sc.labelAnnoMgr.Value(api.XInstanceIdLabelKey)] = targetID + labels[sc.labelAnnoMgr.Value(api.SubResourceTemplateLabelKey)] = template.Name + labels[sc.labelAnnoMgr.Value(api.SubResourceTemplateHashLabelKey)] = template.Hash + resource.SetLabels(labels) + + // Let adapter decorate the resource (optional) + if decorator, ok := adapter.(api.SubResourceDecorator); ok { + if err := decorator.DecorateResource(ctx, xset, template, resource, target, targetID); err != nil { + return fmt.Errorf("failed to decorate %s from template %s: %w", gvk.Kind, template.Name, err) + } + } + + // Set GenerateName if Name is not set + if resource.GetName() == "" { + resource.SetGenerateName(fmt.Sprintf("%s-%s-", xset.GetName(), template.Name)) + } + + if err := sc.client.Create(ctx, resource); err != nil { + if apierrors.IsAlreadyExists(err) { + // Resource already exists — fetch and check state + existingResource, newObjErr := sc.newObjectForGVK(gvk) + if newObjErr != nil { + return fmt.Errorf("failed to create object for GVK %s: %w", gvk, newObjErr) + } + if getErr := sc.client.Get(ctx, client.ObjectKey{ + Namespace: resource.GetNamespace(), + Name: resource.GetName(), + }, existingResource); getErr != nil { + return fmt.Errorf("failed to get existing %s %s: %w", gvk.Kind, resource.GetName(), getErr) + } + // If the existing resource is being deleted, wait for it to be gone + if existingResource.GetDeletionTimestamp() != nil { + // Return error to requeue — the resource will be gone in the next reconcile + return fmt.Errorf("%s %s is being deleted, will retry on next reconcile", gvk.Kind, resource.GetName()) + } + createdResources = append(createdResources, existingResource) + continue + } + return fmt.Errorf("failed to create %s %s: %w", gvk.Kind, resource.GetName(), err) + } + + // Track expectation + if err := sc.expectations.ExpectCreation( + kubeutilclient.ObjectKeyString(xset), + gvk, + resource.GetNamespace(), + resource.GetName(), + ); err != nil { + return err + } + + createdResources = append(createdResources, resource) + } + + // Attach to target (e.g., set volumes on Pod) + if err := adapter.AttachToTarget(ctx, target, createdResources); err != nil { + return fmt.Errorf("failed to attach %s to target: %w", gvk.Kind, err) + } + + return nil +} + +// DeleteTargetResources deletes subresources for a target. +// If isReplaceTarget is true, deletes all resources. Otherwise, only deletes resources +// where the adapter's RetainWhenXSetScaled returns false. +func (sc *RealSubResourceControl) DeleteTargetResources(ctx context.Context, xset api.XSetObject, target client.Object, existing []SubResourceState, isReplaceTarget bool) error { + for _, state := range sc.filterByTarget(existing, target) { + // For replace targets, delete all resources + // For scale-in, only delete if adapter doesn't want to retain + if !isReplaceTarget && state.Adapter != nil && state.Adapter.RetainWhenXSetScaled(xset) { + continue + } + if err := sc.deleteResource(ctx, xset, state.Object, state.GVK); err != nil { + return fmt.Errorf("failed to delete %s %s: %w", state.GVK.Kind, state.Object.GetName(), err) + } + } + return nil +} + +// DeleteTargetUnusedResources deletes unused subresources for a target. +// It classifies resources into new/old by hash and deletes: +// - unclaimed old resources (templates removed from XSet) +// - old resources if not retaining on scale and new version exists +// +// IMPORTANT: This should only be called when the target is being deleted or replaced. +// A template may be removed from the XSet while an existing target still references +// the previously created subresource until that target is recreated or updated. +// Calling this on active targets can delete in-use resources and break workloads. +func (sc *RealSubResourceControl) DeleteTargetUnusedResources(ctx context.Context, xset api.XSetObject, target client.Object, existing []SubResourceState) error { + targetID, exist := sc.labelAnnoMgr.Get(target, api.XInstanceIdLabelKey) + if !exist { + return nil + } + + // Process each adapter type + for _, adapter := range sc.adaptersByGVK { + if err := sc.deleteUnusedResourcesForAdapter(ctx, xset, existing, targetID, adapter); err != nil { + return err + } + } + + return nil +} + +// DeleteTargetRecreateResources deletes subresources for a target that need recreation on update. +// Only deletes resources for adapters where RecreateWhenXSetUpdated returns true. +// This is called in the update phase BEFORE the pod is deleted, so that scale-out in the next +// reconcile creates fresh subresources without cache staleness issues. +func (sc *RealSubResourceControl) DeleteTargetRecreateResources(ctx context.Context, xset api.XSetObject, target client.Object, existing []SubResourceState) error { + for _, state := range sc.filterByTarget(existing, target) { + if state.Adapter == nil || !state.Adapter.RecreateWhenXSetUpdated(xset) { + continue + } + if err := sc.deleteResource(ctx, xset, state.Object, state.GVK); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete %s %s for recreation: %w", state.GVK.Kind, state.Object.GetName(), err) + } + } + return nil +} + +// deleteUnusedResourcesForAdapter handles unused resource deletion for a specific adapter. +func (sc *RealSubResourceControl) deleteUnusedResourcesForAdapter(ctx context.Context, xset api.XSetObject, existing []SubResourceState, targetID string, adapter api.SubResourceAdapter) error { + gvk := adapter.Meta() + + // Classify resources by hash for this adapter + newResources, oldResources, err := sc.classifyResourcesByHash(targetID, xset, adapter, existing) + if err != nil { + return fmt.Errorf("failed to classify %s: %w", gvk.Kind, err) + } + + // Get template names that are in use + templates, err := adapter.GetTemplates(xset) + if err != nil { + return fmt.Errorf("failed to get %s templates: %w", gvk.Kind, err) + } + templateNames := make(map[string]bool) + for _, tmpl := range templates { + templateNames[tmpl.Name] = true + } + + // Delete unclaimed old resources (not in templates) + for templateName, state := range oldResources { + // If resource template is still in use, keep it + if templateNames[templateName] { + continue + } + + if err := sc.deleteResource(ctx, xset, state.Object, state.GVK); err != nil { + return fmt.Errorf("failed to delete unclaimed %s %s: %w", gvk.Kind, state.Object.GetName(), err) + } + } + + // Delete old resources if not retaining on scale and new version exists + if !adapter.RetainWhenXSetScaled(xset) { + for templateName, oldState := range oldResources { + // Only delete if new version exists + if _, hasNew := newResources[templateName]; hasNew { + if err := sc.deleteResource(ctx, xset, oldState.Object, oldState.GVK); err != nil { + return fmt.Errorf("failed to delete old %s %s: %w", gvk.Kind, oldState.Object.GetName(), err) + } + } + } + } + + return nil +} + +// OrphanResource removes owner reference from a subresource. +// This is used when the XSet is being deleted but resources should be retained. +func (sc *RealSubResourceControl) OrphanResource(ctx context.Context, xset api.XSetObject, resource client.Object) error { + xsetSpec := sc.xsetController.GetXSetSpec(xset) + if xsetSpec.Selector.MatchLabels == nil { + return nil + } + + if resource.GetLabels() == nil { + resource.SetLabels(make(map[string]string)) + } + if resource.GetAnnotations() == nil { + resource.SetAnnotations(make(map[string]string)) + } + + refWriter := refmanagerutil.NewOwnerRefWriter(sc.client) + if err := refWriter.Release(ctx, xset, resource); err != nil { + return fmt.Errorf("failed to orphan resource %s: %w", resource.GetName(), err) + } + + return nil +} + +// IsTargetTemplateChanged returns true if any subresource template changed for target, +// or if the target has resources that need recreation on update (when isUpdatedRevision is false). +func (r *RealSubResourceControl) IsTargetTemplateChanged(xset api.XSetObject, target client.Object, existing []SubResourceState, isUpdatedRevision bool) (bool, error) { + targetID, exist := r.labelAnnoMgr.Get(target, api.XInstanceIdLabelKey) + if !exist { + return false, nil + } + + for _, adapter := range r.adaptersByGVK { + changed, err := r.isAdapterTemplateChanged(xset, targetID, adapter, existing) + if err != nil { + return false, err + } + if changed { + return true, nil + } + } + + // Check if any resources need recreation on update (only for non-updated targets) + if !isUpdatedRevision && r.hasRecreateOnUpdateResources(xset, target, existing) { + return true, nil + } + + return false, nil +} + +// isAdapterTemplateChanged checks if template changed for a specific adapter. +// It compares template hashes on existing resources against current template hashes. +// Note: This does NOT check for missing resources (cache lag after creation can cause +// false positives). Missing resource detection is handled by RecreateWhenXSetUpdated +// and the update flow's DeleteTargetRecreateResources. +func (r *RealSubResourceControl) isAdapterTemplateChanged(xset api.XSetObject, targetID string, adapter api.SubResourceAdapter, existing []SubResourceState) (bool, error) { + gvk := adapter.Meta() + + // Get current templates + templates, err := adapter.GetTemplates(xset) + if err != nil { + return false, fmt.Errorf("failed to get templates from adapter %s: %w", gvk.Kind, err) + } + + // Build map of template name to hash + templateHashes := make(map[string]string) + for _, tmpl := range templates { + templateHashes[tmpl.Name] = tmpl.Hash + } + + // Check existing resources for this target and adapter. + // If any existing resource has a hash mismatch or references a removed template, + // the template has changed. + for _, state := range existing { + if state.GVK != gvk { + continue + } + + // Skip resources being deleted + if state.Object.GetDeletionTimestamp() != nil { + continue + } + + // Only check resources for this target + resourceID, exist := r.labelAnnoMgr.Get(state.Object, api.XInstanceIdLabelKey) + if !exist || resourceID != targetID { + continue + } + + // Get template name and hash from the resource + templateName, exist := r.labelAnnoMgr.Get(state.Object, api.SubResourceTemplateLabelKey) + if !exist { + continue + } + + resourceHash, exist := r.labelAnnoMgr.Get(state.Object, api.SubResourceTemplateHashLabelKey) + if !exist { + // No hash means we can't compare, treat as changed + return true, nil + } + + // Check if template still exists + currentHash, templateExists := templateHashes[templateName] + if !templateExists { + // Template was removed, this is a change + return true, nil + } + + // Compare hashes + if currentHash != resourceHash { + return true, nil + } + } + + return false, nil +} + +// CheckAllowIncludeExclude checks if target's subresources allow include/exclude. +// It finds subresources by owner reference + instance ID label and checks each using the provided CheckAllowFunc. +func (r *RealSubResourceControl) CheckAllowIncludeExclude(ctx context.Context, xset api.XSetObject, target client.Object, fn CheckAllowFunc) (bool, error) { + xsetGVK := xset.GetObjectKind().GroupVersionKind() + ownerName := xset.GetName() + ownerKind := xsetGVK.Kind + + // Get all subresources owned by this XSet + resources, err := r.GetFilteredResources(ctx, xset) + if err != nil { + return false, fmt.Errorf("failed to get subresources: %w", err) + } + + // Filter to only those belonging to this target + targetResources := r.filterByTarget(resources, target) + + // Check each subresource + for _, state := range targetResources { + if allowed, reason := fn(state.Object, ownerName, ownerKind, r.labelAnnoMgr); !allowed { + return false, fmt.Errorf("subresource %s/%s does not allow include/exclude: %s", state.Object.GetNamespace(), state.Object.GetName(), reason) + } + } + + return true, nil +} + +// AdoptTargetResources adopts subresources for a target during include operation. +// It finds orphaned resources by selector + orphaned label and adopts those that belong to this target. +func (sc *RealSubResourceControl) AdoptTargetResources(ctx context.Context, xset api.XSetObject, target client.Object, instanceID string) error { + for _, adapter := range sc.adaptersByGVK { + // Find orphaned resources for this adapter + orphaned, err := sc.findOrphanedResourcesForTarget(ctx, xset, adapter, target) + if err != nil { + return fmt.Errorf("failed to find orphaned %s: %w", adapter.Meta().Kind, err) + } + + for _, res := range orphaned { + // Update instance ID and remove orphaned label + sc.labelAnnoMgr.Set(res, api.XInstanceIdLabelKey, instanceID) + sc.labelAnnoMgr.Delete(res, api.XOrphanedIndicationLabelKey) + if err := sc.adoptResource(ctx, xset, res); err != nil { + return err + } + } + } + return nil +} + +// findOrphanedResourcesForTarget finds orphaned subresources that belong to a specific target. +// It uses the PVC adapter (if available) to check which PVCs are mounted to the target. +func (sc *RealSubResourceControl) findOrphanedResourcesForTarget(ctx context.Context, xset api.XSetObject, adapter api.SubResourceAdapter, target client.Object) ([]client.Object, error) { + xsetSpec := sc.xsetController.GetXSetSpec(xset) + ownerSelector := xsetSpec.Selector.DeepCopy() + if ownerSelector.MatchLabels == nil { + ownerSelector.MatchLabels = map[string]string{} + } + ownerSelector.MatchLabels[sc.labelAnnoMgr.Value(api.ControlledByXSetLabel)] = "true" + ownerSelector.MatchExpressions = append(ownerSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: sc.labelAnnoMgr.Value(api.XOrphanedIndicationLabelKey), + Operator: metav1.LabelSelectorOpExists, + }) + + selector, err := metav1.LabelSelectorAsSelector(ownerSelector) + if err != nil { + return nil, err + } + + gvk := adapter.Meta() + list, err := sc.newListForGVK(gvk) + if err != nil { + return nil, nil + } + + if err := sc.client.List(ctx, list, &client.ListOptions{ + Namespace: xset.GetNamespace(), + LabelSelector: selector, + }); err != nil { + return nil, fmt.Errorf("failed to list orphaned %s: %w", gvk.Kind, err) + } + + items := extractListItems(list) + var orphaned []client.Object + for _, item := range items { + // Skip if has owner reference (not truly orphaned) + if len(item.GetOwnerReferences()) > 0 { + continue + } + + // For PVC adapter, check if this PVC is mounted to the target + if gvk.Kind == "PersistentVolumeClaim" { + if pvcAdapter, ok := sc.xsetController.(api.SubResourcePvcAdapter); ok { + volumes := pvcAdapter.GetXSpecVolumes(target) + isMounted := false + for i := range volumes { + if volumes[i].PersistentVolumeClaim != nil && volumes[i].PersistentVolumeClaim.ClaimName == item.GetName() { + isMounted = true + break + } + } + if !isMounted { + continue + } + } + } + + orphaned = append(orphaned, item) + } + return orphaned, nil +} + +// OrphanTargetResources orphans all subresources for a target during exclude operation. +// It finds subresources by owner reference + instance ID label and removes owner reference. +func (sc *RealSubResourceControl) OrphanTargetResources(ctx context.Context, xset api.XSetObject, target client.Object) error { + // Get all subresources owned by this XSet + resources, err := sc.GetFilteredResources(ctx, xset) + if err != nil { + return fmt.Errorf("failed to get subresources: %w", err) + } + + // Filter to only those belonging to this target + targetResources := sc.filterByTarget(resources, target) + + // Orphan each subresource + for _, state := range targetResources { + sc.labelAnnoMgr.Set(state.Object, api.XOrphanedIndicationLabelKey, "true") + if err := sc.OrphanResource(ctx, xset, state.Object); err != nil { + return err + } + } + + return nil +} diff --git a/subresources/subresource_control_test.go b/subresources/subresource_control_test.go new file mode 100644 index 0000000..93f073b --- /dev/null +++ b/subresources/subresource_control_test.go @@ -0,0 +1,78 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package subresources + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kube-xset/api" +) + +func TestSubResourceState(t *testing.T) { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "test-pvc"}, + } + gvk := corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim") + + // Mock adapter for testing + adapter := &mockAdapter{gvk: gvk} + + state := SubResourceState{ + Object: pvc, + GVK: gvk, + Adapter: adapter, + } + + if state.Object.GetName() != "test-pvc" { + t.Errorf("expected name test-pvc, got %s", state.Object.GetName()) + } + if state.GVK.Kind != "PersistentVolumeClaim" { + t.Errorf("expected Kind PersistentVolumeClaim, got %s", state.GVK.Kind) + } +} + +type mockAdapter struct { + gvk schema.GroupVersionKind +} + +func (m *mockAdapter) Meta() schema.GroupVersionKind { return m.gvk } +func (m *mockAdapter) GetTemplates(xset api.XSetObject) ([]api.SubResourceTemplate, error) { + return nil, nil +} +func (m *mockAdapter) RetainWhenXSetDeleted(xset api.XSetObject) bool { return false } +func (m *mockAdapter) RetainWhenXSetScaled(xset api.XSetObject) bool { return false } +func (m *mockAdapter) RecreateWhenXSetUpdated(xset api.XSetObject) bool { return false } +func (m *mockAdapter) AttachToTarget(ctx context.Context, target client.Object, resources []client.Object) error { + return nil +} + +func TestNewRealSubResourceControl(t *testing.T) { + // Test with no adapters + control, err := NewRealSubResourceControl(nil, nil, nil, nil, nil) + if err != nil { + t.Errorf("expected no error with nil adapters, got %v", err) + } + if control != nil { + t.Error("expected nil control for nil adapters") + } +} diff --git a/subresources/utils.go b/subresources/utils.go new file mode 100644 index 0000000..dee1704 --- /dev/null +++ b/subresources/utils.go @@ -0,0 +1,185 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package subresources + +import ( + "encoding/json" + "fmt" + "hash/fnv" + + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// TemplateHash computes a hash of the given template object for change detection. +func TemplateHash(obj interface{}) (string, error) { + bytes, err := json.Marshal(obj) + if err != nil { + return "", fmt.Errorf("failed to marshal template: %w", err) + } + + h := fnv.New32a() + if _, err = h.Write(bytes); err != nil { + return "", fmt.Errorf("failed to compute hash: %w", err) + } + + return rand.SafeEncodeString(fmt.Sprint(h.Sum32())), nil +} + +// ObjectKeyString returns a string representation of namespace/name for logging. +func ObjectKeyString(obj interface { + GetNamespace() string + GetName() string +}, +) string { + if obj.GetNamespace() == "" { + return obj.GetName() + } + return obj.GetNamespace() + "/" + obj.GetName() +} + +// NameTruncator handles resource name truncation within Kubernetes limits. +// It truncates names exceeding the limit and appends a hash suffix for uniqueness. +type NameTruncator struct { + // MaxNameLength is the maximum allowed length for resource names + MaxNameLength int +} + +// NewNameTruncator creates a NameTruncator with default DNS label max length (63). +func NewNameTruncator() *NameTruncator { + return &NameTruncator{ + MaxNameLength: validation.DNS1035LabelMaxLength, // 63 + } +} + +// Truncate truncates name if it exceeds MaxNameLength, appending hash suffix for uniqueness. +func (t *NameTruncator) Truncate(name string) string { + return t.TruncateWithMax(name, t.MaxNameLength) +} + +// TruncateWithMax truncates name to the specified max length with hash suffix. +func (t *NameTruncator) TruncateWithMax(name string, maxLen int) string { + if len(name) <= maxLen { + return name + } + + hash := computeHash(name) + hashSuffix := fmt.Sprintf("-%s", hash) + truncatedLen := maxLen - len(hashSuffix) + + if truncatedLen <= 0 { + // Name too short even for hash, return hash only + return hashSuffix[1:] + } + + return name[:truncatedLen] + hashSuffix +} + +// TruncateLabelValue truncates label value to Kubernetes max (63 chars). +func (t *NameTruncator) TruncateLabelValue(value string) string { + return t.TruncateWithMax(value, validation.LabelValueMaxLength) +} + +// computeHash generates a 6-character hash from the input string. +func computeHash(s string) string { + h := fnv.New32a() + h.Write([]byte(s)) + return rand.SafeEncodeString(fmt.Sprint(h.Sum32()))[:6] +} + +// LabelManager handles setting labels with automatic value truncation. +type LabelManager struct { + truncator *NameTruncator +} + +// NewLabelManager creates a LabelManager with the given truncator. +func NewLabelManager(truncator *NameTruncator) *LabelManager { + return &LabelManager{ + truncator: truncator, + } +} + +// SetLabel sets a label, truncating value if needed with hash suffix. +func (lm *LabelManager) SetLabel(obj client.Object, key, value string) { + if obj.GetLabels() == nil { + obj.SetLabels(make(map[string]string)) + } + + truncatedValue := lm.truncator.TruncateLabelValue(value) + obj.GetLabels()[key] = truncatedValue +} + +// SetLabelWithTrackedOriginal sets a label and tracks the original value in an annotation if truncated. +func (lm *LabelManager) SetLabelWithTrackedOriginal(obj client.Object, key, value string) { + truncatedValue := lm.truncator.TruncateLabelValue(value) + + if obj.GetLabels() == nil { + obj.SetLabels(make(map[string]string)) + } + obj.GetLabels()[key] = truncatedValue + + // Track original value in annotation if truncated + if truncatedValue != value { + if obj.GetAnnotations() == nil { + obj.SetAnnotations(make(map[string]string)) + } + obj.GetAnnotations()[key+".original"] = value + } +} + +// SetOperatingLabel sets an operating label with ID in the key. +// Format: / = timestamp +func (lm *LabelManager) SetOperatingLabel(obj client.Object, prefix, id, value string) { + truncatedID := lm.truncator.TruncateLabelValue(id) + labelKey := fmt.Sprintf("%s/%s", prefix, truncatedID) + + if obj.GetLabels() == nil { + obj.SetLabels(make(map[string]string)) + } + obj.GetLabels()[labelKey] = value +} + +// SetRevisionLabel sets revision info as a label value. +// Key: /, Value: revisionName +func (lm *LabelManager) SetRevisionLabel(obj client.Object, prefix, id, revisionName string) { + truncatedID := lm.truncator.TruncateLabelValue(id) + truncatedRevision := lm.truncator.TruncateLabelValue(revisionName) + + labelKey := fmt.Sprintf("%s/%s", prefix, truncatedID) + + if obj.GetLabels() == nil { + obj.SetLabels(make(map[string]string)) + } + obj.GetLabels()[labelKey] = truncatedRevision + + if truncatedRevision != revisionName { + if obj.GetAnnotations() == nil { + obj.SetAnnotations(make(map[string]string)) + } + obj.GetAnnotations()[labelKey+".original-revision"] = revisionName + } +} + +// GetLabel retrieves a label value from an object. +func (lm *LabelManager) GetLabel(obj client.Object, key string) (string, bool) { + if obj.GetLabels() == nil { + return "", false + } + val, ok := obj.GetLabels()[key] + return val, ok +} diff --git a/subresources/utils_test.go b/subresources/utils_test.go new file mode 100644 index 0000000..1408c99 --- /dev/null +++ b/subresources/utils_test.go @@ -0,0 +1,331 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package subresources + +import ( + "testing" + + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestTemplateHash(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + tests := []struct { + name string + obj interface{} + expectError bool + }{ + { + name: "simple object", + obj: map[string]interface{}{ + "key": "value", + }, + expectError: false, + }, + { + name: "nested object", + obj: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "test", + }, + "spec": map[string]interface{}{ + "replicas": 3, + }, + }, + expectError: false, + }, + { + name: "empty object", + obj: map[string]interface{}{}, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := TemplateHash(tt.obj) + if tt.expectError { + g.Expect(err).To(gomega.HaveOccurred()) + } else { + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(result).ToNot(gomega.BeEmpty()) + } + }) + } +} + +func TestTemplateHash_Deterministic(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + obj := map[string]interface{}{ + "key": "value", + "nested": map[string]interface{}{ + "inner": "data", + }, + } + + hash1, err1 := TemplateHash(obj) + hash2, err2 := TemplateHash(obj) + + g.Expect(err1).ToNot(gomega.HaveOccurred()) + g.Expect(err2).ToNot(gomega.HaveOccurred()) + g.Expect(hash1).To(gomega.Equal(hash2)) +} + +func TestTemplateHash_DifferentObjects(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + obj1 := map[string]interface{}{"key": "value1"} + obj2 := map[string]interface{}{"key": "value2"} + + hash1, err1 := TemplateHash(obj1) + hash2, err2 := TemplateHash(obj2) + + g.Expect(err1).ToNot(gomega.HaveOccurred()) + g.Expect(err2).ToNot(gomega.HaveOccurred()) + g.Expect(hash1).ToNot(gomega.Equal(hash2)) +} + +func TestObjectKeyString(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + tests := []struct { + name string + obj interface { + GetNamespace() string + GetName() string + } + expected string + }{ + { + name: "with namespace", + obj: &mockObject{ + namespace: "default", + name: "my-resource", + }, + expected: "default/my-resource", + }, + { + name: "without namespace", + obj: &mockObject{ + namespace: "", + name: "my-resource", + }, + expected: "my-resource", + }, + { + name: "empty namespace", + obj: &mockObject{ + namespace: "", + name: "test", + }, + expected: "test", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := ObjectKeyString(tt.obj) + g.Expect(result).To(gomega.Equal(tt.expected)) + }) + } +} + +// mockObject is a mock for testing ObjectKeyString +type mockObject struct { + namespace string + name string +} + +func (m *mockObject) GetNamespace() string { return m.namespace } +func (m *mockObject) GetName() string { return m.name } + +func TestNameTruncator_Truncate(t *testing.T) { + g := gomega.NewGomegaWithT(t) + truncator := NewNameTruncator() + + tests := []struct { + name string + input string + expected int // check length, not exact value due to hash + }{ + { + name: "short name unchanged", + input: "short-name", + expected: 10, + }, + { + name: "exactly max length unchanged", + input: "a123456789b123456789c123456789d123456789e123456789f123456789g12", // 63 chars + expected: 63, + }, + { + name: "long name truncated", + input: "this-is-a-very-long-resource-name-that-exceeds-kubernetes-limit-of-63-characters", + expected: 63, + }, + { + name: "very long name truncated", + input: "this-is-an-extremely-long-resource-name-that-is-way-longer-than-any-reasonable-name-should-ever-be", + expected: 63, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := truncator.Truncate(tt.input) + g.Expect(result).To(gomega.HaveLen(tt.expected)) + }) + } +} + +func TestNameTruncator_TruncateWithMax(t *testing.T) { + g := gomega.NewGomegaWithT(t) + truncator := NewNameTruncator() + + // Custom max length + result := truncator.TruncateWithMax("short", 10) + g.Expect(result).To(gomega.Equal("short")) + + result = truncator.TruncateWithMax("this-is-longer-than-ten", 10) + g.Expect(result).To(gomega.HaveLen(10)) +} + +func TestNameTruncator_TruncateLabelValue(t *testing.T) { + g := gomega.NewGomegaWithT(t) + truncator := NewNameTruncator() + + // Short label value unchanged + result := truncator.TruncateLabelValue("short-value") + g.Expect(result).To(gomega.Equal("short-value")) + + // Long label value truncated to 63 + longValue := "this-is-a-very-long-label-value-that-exceeds-kubernetes-limit-of-63-characters-for-labels" + result = truncator.TruncateLabelValue(longValue) + g.Expect(result).To(gomega.HaveLen(63)) +} + +func TestNameTruncator_HashUniqueness(t *testing.T) { + g := gomega.NewGomegaWithT(t) + truncator := NewNameTruncator() + + // Two different long names should produce different truncated results + name1 := "this-is-a-long-resource-name-with-suffix-a" + name2 := "this-is-a-long-resource-name-with-suffix-b" + + result1 := truncator.Truncate(name1) + result2 := truncator.Truncate(name2) + + g.Expect(result1).ToNot(gomega.Equal(result2)) +} + +func TestNameTruncator_Deterministic(t *testing.T) { + g := gomega.NewGomegaWithT(t) + truncator := NewNameTruncator() + + // Same input should produce same output + name := "this-is-a-long-resource-name-for-determinism-test" + + result1 := truncator.Truncate(name) + result2 := truncator.Truncate(name) + + g.Expect(result1).To(gomega.Equal(result2)) +} + +func TestLabelManager_SetLabel(t *testing.T) { + g := gomega.NewGomegaWithT(t) + truncator := NewNameTruncator() + lm := NewLabelManager(truncator) + + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + }, + } + + // Set label on object with no labels + lm.SetLabel(obj, "test-key", "test-value") + g.Expect(obj.Labels["test-key"]).To(gomega.Equal("test-value")) + + // Set label with long value + longValue := "this-is-a-very-long-label-value-that-exceeds-kubernetes-limit-of-63-characters" + lm.SetLabel(obj, "long-key", longValue) + g.Expect(obj.Labels["long-key"]).To(gomega.HaveLen(63)) +} + +func TestLabelManager_SetLabelWithTrackedOriginal(t *testing.T) { + g := gomega.NewGomegaWithT(t) + truncator := NewNameTruncator() + lm := NewLabelManager(truncator) + + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + }, + } + + // Short value - no annotation needed + lm.SetLabelWithTrackedOriginal(obj, "short-key", "short-value") + g.Expect(obj.Labels["short-key"]).To(gomega.Equal("short-value")) + g.Expect(obj.Annotations).To(gomega.BeEmpty()) + + // Long value - annotation tracks original + longValue := "this-is-a-very-long-label-value-that-exceeds-kubernetes-limit-of-63-characters" + lm.SetLabelWithTrackedOriginal(obj, "long-key", longValue) + g.Expect(obj.Labels["long-key"]).To(gomega.HaveLen(63)) + g.Expect(obj.Annotations["long-key.original"]).To(gomega.Equal(longValue)) +} + +func TestLabelManager_SetOperatingLabel(t *testing.T) { + g := gomega.NewGomegaWithT(t) + truncator := NewNameTruncator() + lm := NewLabelManager(truncator) + + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + }, + } + + // Set operating label + lm.SetOperatingLabel(obj, "app.kusionstack.io/operating", "revision-123", "timestamp-value") + + // Check the label key contains truncated ID + expectedKey := "app.kusionstack.io/operating/revision-123" + g.Expect(obj.Labels[expectedKey]).To(gomega.Equal("timestamp-value")) +} + +func TestLabelManager_SetRevisionLabel(t *testing.T) { + g := gomega.NewGomegaWithT(t) + truncator := NewNameTruncator() + lm := NewLabelManager(truncator) + + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + }, + } + + // Set revision label + lm.SetRevisionLabel(obj, "app.kusionstack.io/revision", "id-123", "revision-abc") + + expectedKey := "app.kusionstack.io/revision/id-123" + g.Expect(obj.Labels[expectedKey]).To(gomega.Equal("revision-abc")) +} diff --git a/synccontrols/sync_control.go b/synccontrols/sync_control.go index 3d4ac07..7e48f61 100644 --- a/synccontrols/sync_control.go +++ b/synccontrols/sync_control.go @@ -64,7 +64,7 @@ type SyncControl interface { func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, xsetController api.XSetController, xControl xcontrol.TargetControl, - pvcControl subresources.PvcControl, + subResourceControl subresources.SubResourceControl, xsetLabelAnnoManager api.XSetLabelAnnotationManager, resourceContexts resourcecontexts.ResourceContextControl, cacheExpectations expectations.CacheExpectationsInterface, @@ -94,7 +94,7 @@ func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, xsetLabelAnnoMgr: xsetLabelAnnoManager, resourceContextControl: resourceContexts, xControl: xControl, - pvcControl: pvcControl, + subResourceControl: subResourceControl, updateConfig: updateConfig, cacheExpectations: cacheExpectations, @@ -111,7 +111,7 @@ var _ SyncControl = &RealSyncControl{} type RealSyncControl struct { mixin.ReconcilerMixin xControl xcontrol.TargetControl - pvcControl subresources.PvcControl + subResourceControl subresources.SubResourceControl xsetController api.XSetController xsetLabelAnnoMgr api.XSetLabelAnnotationManager resourceContextControl resourcecontexts.ResourceContextControl @@ -151,19 +151,18 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje return false, nil } - // sync subresource - // 1. list pvcs using ownerReference - // 2. adopt and retain orphaned pvcs according to PVC retention policy - if _, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - var existingPvcs, adoptedPvcs []*corev1.PersistentVolumeClaim - if existingPvcs, err = r.pvcControl.GetFilteredPvcs(ctx, instance); err != nil { - return false, fmt.Errorf("fail to get filtered subresource PVCs: %w", err) + // sync subresources: list owned resources and adopt orphaned ones + if r.subResourceControl != nil { + existing, err := r.subResourceControl.GetFilteredResources(ctx, instance) + if err != nil { + return false, fmt.Errorf("fail to get filtered subresources: %w", err) } - if adoptedPvcs, err = r.pvcControl.AdoptPvcsLeftByRetainPolicy(ctx, instance); err != nil { - return false, fmt.Errorf("fail to adopt orphaned left by whenDelete retention policy PVCs: %w", err) + adopted, err := r.subResourceControl.AdoptOrphanedResources(ctx, instance) + if err != nil { + return false, fmt.Errorf("fail to adopt orphaned subresources: %w", err) } - syncContext.ExistingPvcs = append(syncContext.ExistingPvcs, existingPvcs...) - syncContext.ExistingPvcs = append(syncContext.ExistingPvcs, adoptedPvcs...) + existing = append(existing, adopted...) + syncContext.ExistingSubResources = existing } // sync include exclude targets @@ -227,11 +226,12 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje } } - // delete unused pvcs - if _, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - err = r.pvcControl.DeleteTargetUnusedPvcs(ctx, instance, target, syncContext.ExistingPvcs) - if err != nil { - return false, fmt.Errorf("fail to delete unused pvcs %w", err) + // delete unused subresources only when the target is being removed/recreated. + // Active targets may still reference subresources that are no longer present in the + // latest spec until the target is actually recreated. + if r.subResourceControl != nil && (target.GetDeletionTimestamp() != nil || targetDuringReplace(r.xsetLabelAnnoMgr, target)) { + if err = r.subResourceControl.DeleteTargetUnusedResources(ctx, instance, target, syncContext.ExistingSubResources); err != nil { + return false, fmt.Errorf("fail to delete unused subresources: %w", err) } } @@ -389,31 +389,18 @@ func (r *RealSyncControl) allowIncludeExcludeTargets(ctx context.Context, xset a continue } - // check allowance for subresource - pvcsAllowed := true - if adapter, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - volumes := adapter.GetXSpecVolumes(target) - for i := range volumes { - volume := volumes[i] - if volume.PersistentVolumeClaim == nil { - continue - } - pvc := &corev1.PersistentVolumeClaim{} - err = r.Client.Get(ctx, types.NamespacedName{Namespace: target.GetNamespace(), Name: volume.PersistentVolumeClaim.ClaimName}, pvc) - // if pvc not found, ignore it. In case of pvc is filtered by controller-mesh - if apierrors.IsNotFound(err) { - continue - } else if err != nil { - r.Recorder.Eventf(target, corev1.EventTypeWarning, "ExcludeIncludeNotAllowed", fmt.Sprintf("failed to check allowed to exclude/include from/to xset %s/%s: %s", xset.GetNamespace(), xset.GetName(), err.Error())) - pvcsAllowed = false - } - if allowed, reason := fn(pvc, xset.GetName(), xset.GetObjectKind().GroupVersionKind().Kind, labelMgr); !allowed { - r.Recorder.Eventf(target, corev1.EventTypeWarning, "ExcludeIncludeNotAllowed", fmt.Sprintf("failed to check allowed to exclude/include from/to xset %s/%s: %s", xset.GetNamespace(), xset.GetName(), reason)) - pvcsAllowed = false - } + // check allowance for subresources + subResourcesAllowed := true + if r.subResourceControl != nil { + allowed, checkErr := r.subResourceControl.CheckAllowIncludeExclude(ctx, xset, target, subresources.CheckAllowFunc(fn)) + if checkErr != nil { + r.Recorder.Eventf(target, corev1.EventTypeWarning, "ExcludeIncludeNotAllowed", fmt.Sprintf("failed to check subresource allowed to exclude/include from/to xset %s/%s: %s", xset.GetNamespace(), xset.GetName(), checkErr.Error())) + subResourcesAllowed = false + } else if !allowed { + subResourcesAllowed = false } } - if pvcsAllowed { + if subResourcesAllowed { allowTargets.Insert(targetName) } else { notAllowTargets.Insert(targetName) @@ -575,11 +562,10 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, if err != nil { return apierrors.NewInvalid(schema.GroupKind{Group: r.targetGVK.Group, Kind: r.targetGVK.Kind}, target.GetGenerateName(), []*field.Error{{Detail: err.Error()}}) } - // create pvcs for targets (pod) - if _, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - err = r.pvcControl.CreateTargetPvcs(ctx, xsetObject, target, syncContext.ExistingPvcs) - if err != nil { - return fmt.Errorf("fail to create PVCs for target %s: %w", target.GetName(), err) + // create subresources for targets + if r.subResourceControl != nil { + if err = r.subResourceControl.CreateTargetResources(ctx, xsetObject, target, syncContext.ExistingSubResources); err != nil { + return fmt.Errorf("fail to create subresources for target %s: %w", target.GetName(), err) } } newTarget := target.DeepCopyObject().(client.Object) @@ -604,6 +590,15 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, } r.Recorder.Eventf(xsetObject, corev1.EventTypeNormal, "Scaled", "scale out %d Target(s)", succCount) AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "Scaled", "") + + // Refresh ExistingSubResources after scale-out so that Update phase + // sees newly created subresources and doesn't treat them as missing. + if r.subResourceControl != nil && succCount > 0 { + if refreshed, refreshErr := r.subResourceControl.GetFilteredResources(ctx, xsetObject); refreshErr == nil { + syncContext.ExistingSubResources = refreshed + } + } + return succCount > 0, recordedRequeueAfter, err } } @@ -703,13 +698,12 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, return err } - // delete pvcs if target is in update replace, or retention policy is "Deleted" - if _, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { + // delete subresources if target is in update replace, or retention policy is "Delete" + if r.subResourceControl != nil { _, replaceOrigin := r.xsetLabelAnnoMgr.Get(target.Object, api.XReplacePairOriginName) _, replaceNew := r.xsetLabelAnnoMgr.Get(target.Object, api.XReplacePairNewId) - if replaceOrigin || replaceNew || !r.pvcControl.RetainPvcWhenXSetScaled(xsetObject) { - return r.pvcControl.DeleteTargetPvcs(ctx, xsetObject, target.Object, syncContext.ExistingPvcs) - } + isReplaceTarget := replaceOrigin || replaceNew + return r.subResourceControl.DeleteTargetResources(ctx, xsetObject, target.Object, syncContext.ExistingSubResources, isReplaceTarget) } return nil }) @@ -773,7 +767,7 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, // 3. filter already updated revision, for i, targetInfo := range targetToUpdate { // TODO check decoration and pvc template changed - if targetInfo.IsUpdatedRevision && !targetInfo.PvcTmpHashChanged && !targetInfo.DecorationChanged { + if targetInfo.IsUpdatedRevision && !targetInfo.SubResourceTemplateChanged && !targetInfo.DecorationChanged { continue } @@ -825,6 +819,27 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, "onlyMetadataChanged", targetInfo.OnlyMetadataChanged, ) + // Delete subresources that need recreation before the pod is deleted. + // This ensures fresh subresources are created in the next reconcile's scale-out, + // avoiding cache staleness issues from delete+create in the same reconcile. + if targetInfo.SubResourceTemplateChanged && r.subResourceControl != nil { + if err := r.subResourceControl.DeleteTargetRecreateResources(ctx, xsetObject, targetInfo.Object, syncContext.ExistingSubResources); err != nil { + return err + } + // Remove deleted resources from ExistingSubResources so scale-out + // doesn't try to reuse resources that were just deleted. + targetID, _ := r.xsetLabelAnnoMgr.Get(targetInfo.Object, api.XInstanceIdLabelKey) + filtered := syncContext.ExistingSubResources[:0] + for _, state := range syncContext.ExistingSubResources { + resourceID, _ := r.xsetLabelAnnoMgr.Get(state.Object, api.XInstanceIdLabelKey) + if resourceID == targetID && state.Adapter != nil && state.Adapter.RecreateWhenXSetUpdated(xsetObject) { + continue // skip deleted resource + } + filtered = append(filtered, state) + } + syncContext.ExistingSubResources = filtered + } + spec := r.xsetController.GetXSetSpec(xsetObject) if targetInfo.IsInReplace && spec.UpdateStrategy.UpdatePolicy != api.XSetReplaceTargetUpdateStrategyType { // a replacing target should be replaced by an updated revision target when encountering upgrade @@ -1057,16 +1072,72 @@ func targetDuringReplace(labelMgr api.XSetLabelAnnotationManager, target client. return replaceIndicate || replaceOriginTarget || replaceNewTarget } -// BatchDeleteTargetsByLabel try to trigger target deletion by to-delete label +// BatchDeleteTargetsByLabel triggers target deletion following the same lifecycle pattern as scale-in. +// It triggers TargetOpsLifecycle, waits for permission, then directly deletes the targets. +// Any deletion-related subresource cleanup (including PVC reclamation) is handled separately through +// SubResourceControl/ReclaimSubResourcesOnDeletion rather than in this method. func (r *RealSyncControl) BatchDeleteTargetsByLabel(ctx context.Context, targetControl xcontrol.TargetControl, needDeleteTargets []client.Object) error { + logger := logr.FromContext(ctx) + + // Step 1: Trigger TargetOpsLifecycle for targets not already in lifecycle _, err := controllerutils.SlowStartBatch(len(needDeleteTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { target := needDeleteTargets[i] - if _, exist := r.xsetLabelAnnoMgr.Get(target, api.XDeletionIndicationLabelKey); !exist { - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%d"}}}`, r.xsetLabelAnnoMgr.Value(api.XDeletionIndicationLabelKey), time.Now().UnixNano()))) // nolint - if err := targetControl.PatchTarget(ctx, target, patch); err != nil { - return fmt.Errorf("failed to delete target when syncTargets %s/%s/%w", target.GetNamespace(), target.GetName(), err) + + // Skip if already being deleted + if target.GetDeletionTimestamp() != nil { + return nil + } + + // Check if already during scale-in ops (has preparing-delete label) + if _, duringOps := r.xsetLabelAnnoMgr.Get(target, api.PreparingDeleteLabel); duringOps { + return nil + } + + // Trigger TargetOpsLifecycle with scaleIn OperationType + logger.V(1).Info("try to begin TargetOpsLifecycle for deleting Target in XSet", "target", ObjectKeyString(target)) + if updated, err := opslifecycle.Begin(ctx, r.xsetLabelAnnoMgr, r.Client, r.scaleInLifecycleAdapter, target); err != nil { + return fmt.Errorf("fail to begin TargetOpsLifecycle for deleting Target %s/%s: %w", target.GetNamespace(), target.GetName(), err) + } else if updated { + r.Recorder.Eventf(target, corev1.EventTypeNormal, "BeginDeleteLifecycle", "succeed to begin TargetOpsLifecycle for deletion") + // add an expectation for this target update, before next reconciling + if err := r.cacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(target), r.targetGVK, target.GetNamespace(), target.GetName(), target.GetResourceVersion()); err != nil { + return err } } + + return nil + }) + if err != nil { + return err + } + + // Step 2: Check AllowOps and delete targets that are allowed + _, err = controllerutils.SlowStartBatch(len(needDeleteTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { + target := needDeleteTargets[i] + + // Skip if already being deleted + if target.GetDeletionTimestamp() != nil { + return nil + } + + // Check if operation is allowed (no delay for deletion, pass 0) + _, allowed := opslifecycle.AllowOps(r.xsetLabelAnnoMgr, r.scaleInLifecycleAdapter, 0, target) + if !allowed { + logger.V(1).Info("target not yet allowed to delete, waiting for lifecycle", "target", ObjectKeyString(target)) + return nil + } + + // Delete the target + logger.Info("deleting target for XSet deletion", "target", ObjectKeyString(target)) + if err := targetControl.DeleteTarget(ctx, target); err != nil { + return fmt.Errorf("failed to delete target %s/%s: %w", target.GetNamespace(), target.GetName(), err) + } + + r.Recorder.Eventf(target, corev1.EventTypeNormal, "TargetDeleted", "succeed to delete target for XSet deletion") + if err := r.cacheExpectations.ExpectDeletion(clientutil.ObjectKeyString(target), r.targetGVK, target.GetNamespace(), target.GetName()); err != nil { + return err + } + return nil }) return err diff --git a/synccontrols/types.go b/synccontrols/types.go index d4b0cf4..95a779e 100644 --- a/synccontrols/types.go +++ b/synccontrols/types.go @@ -20,18 +20,20 @@ import ( "time" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" "kusionstack.io/kube-xset/api" + "kusionstack.io/kube-xset/subresources" ) type SyncContext struct { - Revisions []*appsv1.ControllerRevision - CurrentRevision *appsv1.ControllerRevision - UpdatedRevision *appsv1.ControllerRevision - ExistingSubResource []client.Object + Revisions []*appsv1.ControllerRevision + CurrentRevision *appsv1.ControllerRevision + UpdatedRevision *appsv1.ControllerRevision + + // ExistingSubResources holds all subresources owned by the XSet, populated in SyncTargets. + ExistingSubResources []subresources.SubResourceState FilteredTarget []client.Object TargetWrappers []*TargetWrapper @@ -41,15 +43,9 @@ type SyncContext struct { CurrentIDs sets.Int OwnedIds map[int]*api.ContextDetail - SubResources - NewStatus *api.XSetStatus } -type SubResources struct { - ExistingPvcs []*corev1.PersistentVolumeClaim -} - type TargetWrapper struct { // parameters must be set during creation client.Object @@ -112,6 +108,6 @@ type TargetUpdateInfo struct { } type SubResourcesChanged struct { - // indicate if the pvc template changed - PvcTmpHashChanged bool + // indicate if any subresource template changed + SubResourceTemplateChanged bool } diff --git a/synccontrols/x_replace.go b/synccontrols/x_replace.go index 71951d4..f95412c 100644 --- a/synccontrols/x_replace.go +++ b/synccontrols/x_replace.go @@ -36,7 +36,6 @@ import ( "kusionstack.io/kube-xset/api" "kusionstack.io/kube-xset/opslifecycle" - "kusionstack.io/kube-xset/subresources" "kusionstack.io/kube-xset/xcontrol" ) @@ -177,11 +176,10 @@ func (r *RealSyncControl) replaceOriginTargets( r.xsetLabelAnnoMgr.Set(newTarget, api.XCreatingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) r.resourceContextControl.Put(newTargetContext, api.EnumRevisionContextDataKey, replaceRevision.GetName()) - // create pvcs for new target - if _, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - err = r.pvcControl.CreateTargetPvcs(ctx, instance, newTarget, syncContext.ExistingPvcs) - if err != nil { - return fmt.Errorf("fail to create PVCs for target %s: %w", newTarget.GetName(), err) + // create subresources for new target + if r.subResourceControl != nil { + if err = r.subResourceControl.CreateTargetResources(ctx, instance, newTarget, syncContext.ExistingSubResources); err != nil { + return fmt.Errorf("fail to create subresources for target %s: %w", newTarget.GetName(), err) } } diff --git a/synccontrols/x_scale.go b/synccontrols/x_scale.go index 0dec0ae..e299e87 100644 --- a/synccontrols/x_scale.go +++ b/synccontrols/x_scale.go @@ -23,8 +23,6 @@ import ( "strconv" "time" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" @@ -37,7 +35,6 @@ import ( "kusionstack.io/kube-xset/api" "kusionstack.io/kube-xset/features" "kusionstack.io/kube-xset/opslifecycle" - "kusionstack.io/kube-xset/subresources" "kusionstack.io/kube-xset/xcontrol" ) @@ -168,27 +165,10 @@ func (r *RealSyncControl) excludeTarget(ctx context.Context, xsetObject api.XSet return err } - // exclude subresource - if adapter, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - volumes := adapter.GetXSpecVolumes(target) - for i := range volumes { - volume := volumes[i] - if volume.PersistentVolumeClaim == nil { - continue - } - pvc := &corev1.PersistentVolumeClaim{} - err := r.Client.Get(ctx, types.NamespacedName{Namespace: target.GetNamespace(), Name: volume.PersistentVolumeClaim.ClaimName}, pvc) - // If pvc not found, ignore it. In case of pvc is filtered out by controller-mesh - if apierrors.IsNotFound(err) { - continue - } else if err != nil { - return err - } - - r.xsetLabelAnnoMgr.Set(pvc, api.XOrphanedIndicationLabelKey, "true") - if err := r.pvcControl.OrphanPvc(ctx, xsetObject, pvc); err != nil { - return err - } + // exclude subresources + if r.subResourceControl != nil { + if err := r.subResourceControl.OrphanTargetResources(ctx, xsetObject, target); err != nil { + return err } } @@ -206,28 +186,10 @@ func (r *RealSyncControl) includeTarget(ctx context.Context, xsetObject api.XSet return err } - // exclude subresource - if adapter, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - volumes := adapter.GetXSpecVolumes(target) - for i := range volumes { - volume := volumes[i] - if volume.PersistentVolumeClaim == nil { - continue - } - pvc := &corev1.PersistentVolumeClaim{} - err := r.Client.Get(ctx, types.NamespacedName{Namespace: target.GetNamespace(), Name: volume.PersistentVolumeClaim.ClaimName}, pvc) - // If pvc not found, ignore it. In case of pvc is filtered out by controller-mesh - if apierrors.IsNotFound(err) { - continue - } else if err != nil { - return err - } - - r.xsetLabelAnnoMgr.Set(pvc, api.XInstanceIdLabelKey, instanceId) - r.xsetLabelAnnoMgr.Delete(pvc, api.XOrphanedIndicationLabelKey) - if err := r.pvcControl.AdoptPvc(ctx, xsetObject, pvc); err != nil { - return err - } + // include subresources + if r.subResourceControl != nil { + if err := r.subResourceControl.AdoptTargetResources(ctx, xsetObject, target, instanceId); err != nil { + return err } } diff --git a/synccontrols/x_update.go b/synccontrols/x_update.go index 1faf97f..f7264d4 100644 --- a/synccontrols/x_update.go +++ b/synccontrols/x_update.go @@ -38,7 +38,6 @@ import ( "kusionstack.io/kube-xset/api" "kusionstack.io/kube-xset/opslifecycle" "kusionstack.io/kube-xset/resourcecontexts" - "kusionstack.io/kube-xset/subresources" "kusionstack.io/kube-xset/xcontrol" ) @@ -89,9 +88,9 @@ func (r *RealSyncControl) attachTargetUpdateInfo(_ context.Context, xsetObject a spec := r.xsetController.GetXSetSpec(xsetObject) // decide whether the TargetOpsLifecycle is during ops or not updateInfo.RequeueForOperationDelay, updateInfo.IsAllowUpdateOps = opslifecycle.AllowOps(r.updateConfig.XsetLabelAnnoMgr, r.updateLifecycleAdapter, ptr.Deref(spec.UpdateStrategy.OperationDelaySeconds, 0), target) - // check subresource pvc template changed - if _, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - updateInfo.PvcTmpHashChanged, err = r.pvcControl.IsTargetPvcTmpChanged(xsetObject, target.Object, syncContext.ExistingPvcs) + // check subresource template changed + if r.subResourceControl != nil { + updateInfo.SubResourceTemplateChanged, err = r.subResourceControl.IsTargetTemplateChanged(xsetObject, target.Object, syncContext.ExistingSubResources, updateInfo.IsUpdatedRevision) if err != nil { return nil, err } @@ -438,7 +437,7 @@ func (u *GenericTargetUpdater) FilterAllowOpsTargets(ctx context.Context, candid targetInfo.IsAllowUpdateOps = true - if targetInfo.IsUpdatedRevision && !targetInfo.PvcTmpHashChanged && !targetInfo.DecorationChanged { + if targetInfo.IsUpdatedRevision && !targetInfo.SubResourceTemplateChanged && !targetInfo.DecorationChanged { continue } @@ -586,7 +585,7 @@ func (u *replaceUpdateTargetUpdater) BeginUpdateTarget(ctx context.Context, sync func (u *replaceUpdateTargetUpdater) FilterAllowOpsTargets(_ context.Context, candidates []*TargetUpdateInfo, _ map[int]*api.ContextDetail, _ *SyncContext, targetCh chan *TargetUpdateInfo) (requeueAfter *time.Duration, err error) { activeTargetToUpdate := filterOutPlaceHolderUpdateInfos(candidates) for i, targetInfo := range activeTargetToUpdate { - if targetInfo.IsUpdatedRevision && !targetInfo.PvcTmpHashChanged && !targetInfo.DecorationChanged { + if targetInfo.IsUpdatedRevision && !targetInfo.SubResourceTemplateChanged && !targetInfo.DecorationChanged { continue } diff --git a/synccontrols/x_utils.go b/synccontrols/x_utils.go index 920928c..2a6ce80 100644 --- a/synccontrols/x_utils.go +++ b/synccontrols/x_utils.go @@ -42,7 +42,13 @@ func NewTargetFrom(setController api.XSetController, xsetLabelAnnoMgr api.XSetLa ownerRef := metav1.NewControllerRef(owner, meta.GroupVersionKind()) targetObj.SetOwnerReferences(append(targetObj.GetOwnerReferences(), *ownerRef)) targetObj.SetNamespace(owner.GetNamespace()) - targetObj.SetGenerateName(GetTargetsPrefix(owner.GetName())) + + // Get prefix from controller (may be empty for default) + var prefixOverride string + if pg, ok := setController.(api.TargetPrefixGetter); ok { + prefixOverride = pg.GetTargetPrefix(owner) + } + targetObj.SetGenerateName(GetTargetsPrefix(prefixOverride, owner.GetName())) if IsTargetNamingSuffixPolicyPersistentSequence(setController.GetXSetSpec(owner)) { targetObj.SetName(fmt.Sprintf("%s%d", targetObj.GetGenerateName(), id)) @@ -99,7 +105,12 @@ func AddOrUpdateCondition(status *api.XSetStatus, conditionType api.XSetConditio } } -func GetTargetsPrefix(controllerName string) string { +// GetTargetsPrefix returns the prefix for target names. +// If override is non-empty, uses it; otherwise uses controllerName with DNS validation. +func GetTargetsPrefix(override, controllerName string) string { + if override != "" { + return override + } // use the dash (if the name isn't too long) to make the target name a bit prettier prefix := fmt.Sprintf("%s-", controllerName) if len(apimachineryvalidation.NameIsDNSSubdomain(prefix, true)) != 0 { diff --git a/synccontrols/x_utils_test.go b/synccontrols/x_utils_test.go new file mode 100644 index 0000000..1e93932 --- /dev/null +++ b/synccontrols/x_utils_test.go @@ -0,0 +1,62 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package synccontrols + +import "testing" + +func TestGetTargetsPrefix(t *testing.T) { + tests := []struct { + name string + override string + controllerName string + expected string + }{ + { + name: "default naming", + override: "", + controllerName: "myset", + expected: "myset-", + }, + { + name: "custom prefix", + override: "custom-", + controllerName: "myset", + expected: "custom-", + }, + { + name: "custom prefix without dash", + override: "custom", + controllerName: "myset", + expected: "custom", + }, + { + name: "empty override uses default", + override: "", + controllerName: "test-controller", + expected: "test-controller-", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := GetTargetsPrefix(tt.override, tt.controllerName) + if result != tt.expected { + t.Errorf("GetTargetsPrefix() = %v, want %v", result, tt.expected) + } + }) + } +} diff --git a/xset_controller.go b/xset_controller.go index 9c34ce3..3a01484 100644 --- a/xset_controller.go +++ b/xset_controller.go @@ -61,7 +61,7 @@ type xSetCommonReconciler struct { // reconcile logic helpers cacheExpectations *expectations.CacheExpectations targetControl xcontrol.TargetControl - pvcControl subresources.PvcControl + subResourceControl subresources.SubResourceControl syncControl synccontrols.SyncControl revisionManager history.HistoryManager resourceContextControl resourcecontexts.ResourceContextControl @@ -90,11 +90,14 @@ func SetUpWithManager(mgr ctrl.Manager, xsetController api.XSetController) error } cacheExpectations := expectations.NewxCacheExpectations(reconcilerMixin.Client, reconcilerMixin.Scheme, clock.RealClock{}) resourceContextControl := resourcecontexts.NewRealResourceContextControl(reconcilerMixin, xsetController, resourceContextAdapter, resourceContextGVK, cacheExpectations, xsetLabelManager) - pvcControl, err := subresources.NewRealPvcControl(reconcilerMixin, cacheExpectations, xsetLabelManager, xsetController) + adapters := subresources.BuildAdapters(xsetController, xsetLabelManager) + subResourceControl, err := subresources.NewRealSubResourceControl( + reconcilerMixin, adapters, cacheExpectations, xsetLabelManager, xsetController, + ) if err != nil { - return errors.New("failed to create pvc control") + return fmt.Errorf("failed to create subresource control: %w", err) } - syncControl := synccontrols.NewRealSyncControl(reconcilerMixin, xsetController, targetControl, pvcControl, xsetLabelManager, resourceContextControl, cacheExpectations) + syncControl := synccontrols.NewRealSyncControl(reconcilerMixin, xsetController, targetControl, subResourceControl, xsetLabelManager, resourceContextControl, cacheExpectations) revisionControl := history.NewRevisionControl(reconcilerMixin.Client, reconcilerMixin.Client) revisionOwner := revisionowner.NewRevisionOwner(xsetController, targetControl) revisionManager := history.NewHistoryManager(revisionControl, revisionOwner) @@ -105,7 +108,7 @@ func SetUpWithManager(mgr ctrl.Manager, xsetController api.XSetController) error XSetController: xsetController, meta: xsetController.XSetMeta(), finalizerName: xsetController.FinalizerName(), - pvcControl: pvcControl, + subResourceControl: subResourceControl, syncControl: syncControl, revisionManager: revisionManager, resourceContextControl: resourceContextControl, @@ -302,39 +305,10 @@ func (r *xSetCommonReconciler) releaseResourcesForDeletion(ctx context.Context, } func (r *xSetCommonReconciler) ensureReclaimTargetSubResources(ctx context.Context, xset api.XSetObject) error { - if _, enabled := subresources.GetSubresourcePvcAdapter(r.XSetController); enabled { - err := r.ensureReclaimPvcs(ctx, xset) - if err != nil { - return err - } - } - return nil -} - -// ensureReclaimPvcs removes xset ownerReference from pvcs if RetainPvcWhenXSetDeleted. -// This allows pvcs to be retained for other xsets with same pvc template. -func (r *xSetCommonReconciler) ensureReclaimPvcs(ctx context.Context, xset api.XSetObject) error { - if !r.pvcControl.RetainPvcWhenXSetDeleted(xset) { + if r.subResourceControl == nil { return nil } - var needReclaimPvcs []*corev1.PersistentVolumeClaim - pvcs, err := r.pvcControl.GetFilteredPvcs(ctx, xset) - if err != nil { - return err - } - // reclaim pvcs if RetainPvcWhenXSetDeleted - for i := range pvcs { - owned := pvcs[i].OwnerReferences != nil && len(pvcs[i].OwnerReferences) > 0 - if owned { - needReclaimPvcs = append(needReclaimPvcs, pvcs[i]) - } - } - for i := range needReclaimPvcs { - if err := r.pvcControl.OrphanPvc(ctx, xset, needReclaimPvcs[i]); err != nil { - return err - } - } - return nil + return r.subResourceControl.ReclaimSubResourcesOnDeletion(ctx, xset) } func (r *xSetCommonReconciler) ensureReclaimTargetsDeletion(ctx context.Context, instance api.XSetObject) (bool, error) {