Refactor experiment signals (#2944)

* refactor

* review fixes

* Review suggestions

* add cluster uid tag

* Fix version check

* Update default versions

* separate goroutine for acks

* Review suggestions

* Simplify refactor

* fix go.mod

* skip checking experiment ID on promote signal

* exclude fleet.datadoghq.com annotation from controller revision

---------

Co-authored-by: Paul Coignet <paul.coignet@datadoghq.com>
Co-authored-by: levan-m <116471169+levan-m@users.noreply.github.com>
Co-authored-by: Levan Machablishvili <levan.machablishvili@datadoghq.com>

Author: 3 people

api/datadoghq/v2alpha1/const.go

@@ -13,3 +13,29 @@ const (
 	// DefaultAPIKeyKey default api-key key (use in secret for instance).
 	DefaultAPIKeyKey = "api_key"
 )
+
+// Experiment signal annotations. The fleet daemon writes these annotations to
+// request state transitions; the reconciler clears them after processing.
+const (
+	// AnnotationExperimentID is the annotation key for the experiment signal ID.
+	AnnotationExperimentID = "experiment.datadoghq.com/id"
+	// AnnotationExperimentSignal is the annotation key for the experiment signal type.
+	AnnotationExperimentSignal = "experiment.datadoghq.com/signal"
+)
+
+// Fleet pending-operation annotations. The fleet daemon writes these
+// annotations to durably track the single in-flight async operation for a DDA
+// so it can recover after restarts.
+const (
+	// AnnotationPendingTaskID is the RC updater task ID currently awaiting completion.
+	AnnotationPendingTaskID = "fleet.datadoghq.com/pending-task-id"
+	// AnnotationPendingAction is the pending fleet action, such as start/stop/promote.
+	AnnotationPendingAction = "fleet.datadoghq.com/pending-action"
+	// AnnotationPendingExperimentID is the stable experiment identity (`params.version`).
+	AnnotationPendingExperimentID = "fleet.datadoghq.com/pending-experiment-id"
+	// AnnotationPendingPackage is the RC package whose Task.State/config versions should be updated.
+	AnnotationPendingPackage = "fleet.datadoghq.com/pending-package"
+	// AnnotationPendingResultVersion is the RC config version to write on
+	// success when it differs from the experiment identity, such as promote.
+	AnnotationPendingResultVersion = "fleet.datadoghq.com/pending-result-version"
+)

api/datadoghq/v2alpha1/datadogagent_types.go

@@ -2427,33 +2427,45 @@ type RemoteConfigConfiguration struct {
 }
 
 // ExperimentPhase is the lifecycle phase of a Fleet Automation experiment.
-// +kubebuilder:validation:Enum=running;stopped;rollback;timeout;promoted;aborted
+// +kubebuilder:validation:Enum=running;terminated;promoted;aborted
 type ExperimentPhase string
 
 const (
-	// ExperimentPhaseRunning is set by RC when an experiment starts (startExperiment).
+	// ExperimentPhaseRunning is set by the reconciler when it processes a start signal.
 	ExperimentPhaseRunning ExperimentPhase = "running"
-	// ExperimentPhaseStopped is set by RC to request a rollback (stopExperiment).
-	ExperimentPhaseStopped ExperimentPhase = "stopped"
-	// ExperimentPhaseRollback is set by the operator after processing a stopped signal and restoring the previous spec.
-	ExperimentPhaseRollback ExperimentPhase = "rollback"
-	// ExperimentPhaseTimeout is set by the operator when the experiment exceeds the timeout and is auto-rolled back.
-	ExperimentPhaseTimeout ExperimentPhase = "timeout"
-	// ExperimentPhasePromoted is set by RC when an experiment succeeds (promoteExperiment).
+	// ExperimentPhaseTerminated is set by the reconciler after restoring the previous spec,
+	// either due to an explicit rollback signal or a timeout. The TerminationReason field
+	// distinguishes the cause.
+	ExperimentPhaseTerminated ExperimentPhase = "terminated"
+	// ExperimentPhasePromoted is set by the reconciler when a promote signal is processed.
 	ExperimentPhasePromoted ExperimentPhase = "promoted"
-	// ExperimentPhaseAborted is set by the operator when a manual spec change is detected during a running experiment.
+	// ExperimentPhaseAborted is set by the reconciler when a manual spec change is detected during a running experiment.
 	ExperimentPhaseAborted ExperimentPhase = "aborted"
 )
 
+// Experiment signal values written to the AnnotationExperimentSignal annotation.
+const (
+	// ExperimentSignalStart requests a new experiment to begin.
+	ExperimentSignalStart = "start"
+	// ExperimentSignalRollback requests the current experiment to roll back.
+	ExperimentSignalRollback = "rollback"
+	// ExperimentSignalPromote requests the current experiment to be promoted.
+	ExperimentSignalPromote = "promote"
+)
+
 // ExperimentStatus defines the state of a Fleet Automation experiment.
 // +k8s:openapi-gen=true
 type ExperimentStatus struct {
 	// Phase is the current state of the experiment.
 	// +optional
 	Phase ExperimentPhase `json:"phase,omitempty"`
-	// ID is the unique experiment ID sent by Fleet Automation.
+	// ID is the RC task ID that triggered this experiment state.
 	// +optional
 	ID string `json:"id,omitempty"`
+	// TerminationReason distinguishes why the experiment was terminated.
+	// Only set when Phase is "terminated".
+	// +optional
+	TerminationReason string `json:"terminationReason,omitempty"`
 }
 
 // DatadogAgentStatus defines the observed state of DatadogAgent.

api/datadoghq/v2alpha1/zz_generated.openapi.go

@@ -651,6 +651,6 @@ func setupAndStartHelmMetadataForwarder(logger logr.Logger, mgr manager.Manager,
 }
 
 func setupFleetDaemon(logger logr.Logger, mgr manager.Manager, rcClient remoteconfig.RCClient, revisionsEnabled bool) error {
-	daemon := fleet.NewDaemon(rcClient, mgr.GetClient(), revisionsEnabled)
+	daemon := fleet.NewDaemon(rcClient, mgr, revisionsEnabled)
 	return mgr.Add(daemon)
 }

cmd/main.go

@@ -8767,18 +8767,21 @@ spec:
                   description: Experiment tracks the state of an active or recent Fleet Automation experiment.
                   properties:
                     id:
-                      description: ID is the unique experiment ID sent by Fleet Automation.
+                      description: ID is the RC task ID that triggered this experiment state.
                       type: string
                     phase:
                       description: Phase is the current state of the experiment.
                       enum:
                         - running
-                        - stopped
-                        - rollback
-                        - timeout
+                        - terminated
                         - promoted
                         - aborted
                       type: string
+                    terminationReason:
+                      description: |-
+                        TerminationReason distinguishes why the experiment was terminated.
+                        Only set when Phase is "terminated".
+                      type: string
                   type: object
                 otelAgentGateway:
                   description: The actual state of the OTel Agent Gateway as a deployment.

config/crd/bases/v1/datadoghq.com_datadogagents.yaml

@@ -8456,20 +8456,22 @@
           "description": "Experiment tracks the state of an active or recent Fleet Automation experiment.",
           "properties": {
             "id": {
-              "description": "ID is the unique experiment ID sent by Fleet Automation.",
+              "description": "ID is the RC task ID that triggered this experiment state.",
               "type": "string"
             },
             "phase": {
               "description": "Phase is the current state of the experiment.",
               "enum": [
                 "running",
-                "stopped",
-                "rollback",
-                "timeout",
+                "terminated",
                 "promoted",
                 "aborted"
               ],
               "type": "string"
+            },
+            "terminationReason": {
+              "description": "TerminationReason distinguishes why the experiment was terminated.\nOnly set when Phase is \"terminated\".",
+              "type": "string"
             }
           },
           "type": "object"

config/crd/bases/v1/datadoghq.com_datadogagents_v2alpha1.json

@@ -68,6 +68,7 @@ require (
 	github.com/samber/lo v1.52.0
 	golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8
 	golang.org/x/text v0.35.0
+	google.golang.org/protobuf v1.36.11
 	helm.sh/helm/v3 v3.20.2
 	k8s.io/kubectl v0.35.3
 	k8s.io/utils v0.0.0-20251222233032-718f0e51e6d2
@@ -280,7 +281,6 @@ require (
 	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
 	google.golang.org/grpc v1.79.3 // indirect
-	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gotest.tools/v3 v3.5.1 // indirect