diff --git a/Dockerfile b/Dockerfile index 19de48a7..4a83f6e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ COPY go.sum go.sum RUN go mod download # Copy the go source -COPY cmd/main.go cmd/main.go +COPY cmd/ cmd/ COPY api/ api/ COPY internal/ internal/ COPY pkg/ pkg/ @@ -23,12 +23,14 @@ COPY pkg/ pkg/ # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o mirror-agent ./cmd/mirror-agent # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details FROM gcr.io/distroless/static:nonroot@sha256:963fa6c544fe5ce420f1f54fb88b6fb01479f054c8056d0f74cc2c6000df5240 WORKDIR / COPY --from=builder /workspace/manager . +COPY --from=builder /workspace/mirror-agent . USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index 05d9ad1d..d6249376 100644 --- a/Makefile +++ b/Makefile @@ -86,9 +86,10 @@ test: manifests generate fmt vet envtest ## Run tests. # Prometheus and CertManager are installed by default; skip with: # - PROMETHEUS_INSTALL_SKIP=true # - CERT_MANAGER_INSTALL_SKIP=true +# Explicit -timeout: the features' worst-case wait ceilings sum past the go-test default of 10m. .PHONY: test-e2e test-e2e: generate fmt vet kind gofail-enable ## Run the e2e tests. Expected an isolated environment using Kind. - ETCD_VERSION="$(E2E_ETCD_VERSION)" PATH="$(LOCALBIN):$(PATH)" go test ./test/e2e/ -v + ETCD_VERSION="$(E2E_ETCD_VERSION)" PATH="$(LOCALBIN):$(PATH)" go test ./test/e2e/ -v -timeout 30m $(MAKE) gofail-disable .PHONY: gofail-enable @@ -125,8 +126,9 @@ verify: verify-mod-tidy lint ## Run static checks against the code. ##@ Build .PHONY: build -build: manifests generate fmt vet ## Build manager binary. +build: manifests generate fmt vet ## Build manager and mirror-agent binaries. go build -o bin/manager cmd/main.go + go build -o bin/mirror-agent ./cmd/mirror-agent .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. diff --git a/PROJECT b/PROJECT index f70bcb68..c196e1ea 100644 --- a/PROJECT +++ b/PROJECT @@ -17,4 +17,13 @@ resources: kind: EtcdCluster path: go.etcd.io/etcd-operator/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: etcd.io + group: operator + kind: EtcdMirror + path: go.etcd.io/etcd-operator/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/api/v1alpha1/etcdmirror_types.go b/api/v1alpha1/etcdmirror_types.go new file mode 100644 index 00000000..528a6e95 --- /dev/null +++ b/api/v1alpha1/etcdmirror_types.go @@ -0,0 +1,953 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EtcdMirrorMode selects the mirror's operating mode. +type EtcdMirrorMode string + +const ( + // EtcdMirrorModeSync is normal continuous replication. + EtcdMirrorModeSync EtcdMirrorMode = "Sync" + // EtcdMirrorModeDrain prepares for cutover: the agent records the source + // revision observed when Drain is requested (status.cutover.drainTargetRevision), + // keeps replicating until the checkpoint watermark reaches it, runs a + // verification pass (per-side key counts, lease-backed key count), then + // sets the CutoverReady condition and flips the fence key's role to + // Primary so any straggler apply fails its mod-revision compare loudly. + // Runbook: quiesce source writers -> set mode=Drain -> + // `kubectl wait --for=condition=CutoverReady etcdmirror/` -> + // purge/re-lease lease-backed keys -> repoint clients -> delete the CR. + EtcdMirrorModeDrain EtcdMirrorMode = "Drain" +) + +// EtcdMirrorInitialSyncMode governs how the agent treats pre-existing keys +// under the effective destination prefix at genesis (first-ever sync, or a +// checkpoint invalidated by a cluster-identity mismatch). +type EtcdMirrorInitialSyncMode string + +const ( + // EtcdMirrorInitialSyncRequireEmpty refuses to start if the destination + // prefix already holds any key (Phase -> Failed, condition + // EmptyTargetViolation). The reserved checkpoint key is excluded by exact + // match. + // + // RE-ARM CONTRACT: a source OR target cluster-ID mismatch invalidates + // the checkpoint, forces genesis, and RE-ARMS this check. An ordinary + // forced resync (Compacted) does NOT re-check RequireEmpty: the decoded, + // ownership-validated fence proves the destination data is this link's + // own. + EtcdMirrorInitialSyncRequireEmpty EtcdMirrorInitialSyncMode = "RequireEmpty" + // EtcdMirrorInitialSyncOverwrite scans and writes over whatever is there. + // Keys present on the target but absent on the source are left alone. + EtcdMirrorInitialSyncOverwrite EtcdMirrorInitialSyncMode = "Overwrite" + // EtcdMirrorInitialSyncOverwriteAndPrune is Overwrite plus one mandatory + // orphan-prune pass after the scan: target keys under the destination + // prefix with no source counterpart are deleted. This makes reversal onto + // a previously-populated prefix (failback) a first-class correct + // operation instead of silently resurrecting deleted keys. + EtcdMirrorInitialSyncOverwriteAndPrune EtcdMirrorInitialSyncMode = "OverwriteAndPrune" +) + +// EtcdMirrorSpec defines the desired state of an EtcdMirror. +// +// Range-defining and rewrite fields are immutable (CEL transition rules +// below): source.prefix, target.prefix, sync.destPrefix, sync.excludePrefixes +// and checkpoint.key. Changing what range is mirrored, or where it lands, +// mid-life silently diverges: a restarted agent resumes from its checkpoint +// without a scan, so removing an exclusion never backfills pre-existing keys +// and adding one strands already-mirrored keys as permanent orphans — all +// with every condition green. Endpoints stay mutable — rotating an NLB DNS +// name or adding a member to the same cluster is routine; pointing at a +// different cluster is caught at runtime by the checkpoint's dual-cluster-ID +// binding, not by spec validation. +// +// The transition rules compare VALUES, presence-normalized: for these +// fields the empty value and an absent field are semantically identical +// (prefix "" = whole keyspace, destPrefix "" = strip the source prefix), and +// Go typed clients drop explicit "" through omitempty — presence-based rules +// would reject every typed-client update of a CR created with an explicit +// empty string. +// +// +kubebuilder:validation:XValidation:rule="(has(self.source.prefix) ? self.source.prefix : \"\") == (has(oldSelf.source.prefix) ? oldSelf.source.prefix : \"\")",message="source.prefix is immutable" +// +kubebuilder:validation:XValidation:rule="(has(self.target.prefix) ? self.target.prefix : \"\") == (has(oldSelf.target.prefix) ? oldSelf.target.prefix : \"\")",message="target.prefix is immutable" +// +kubebuilder:validation:XValidation:rule="(has(self.sync) && has(self.sync.destPrefix) ? self.sync.destPrefix : \"\") == (has(oldSelf.sync) && has(oldSelf.sync.destPrefix) ? oldSelf.sync.destPrefix : \"\")",message="sync.destPrefix is immutable" +// +kubebuilder:validation:XValidation:rule="(has(self.sync) && has(self.sync.excludePrefixes) ? self.sync.excludePrefixes : []) == (has(oldSelf.sync) && has(oldSelf.sync.excludePrefixes) ? oldSelf.sync.excludePrefixes : [])",message="sync.excludePrefixes is immutable" +// +kubebuilder:validation:XValidation:rule="(has(self.checkpoint) && has(self.checkpoint.key) ? self.checkpoint.key : \"\") == (has(oldSelf.checkpoint) && has(oldSelf.checkpoint.key) ? oldSelf.checkpoint.key : \"\")",message="checkpoint.key is immutable" +// +kubebuilder:validation:XValidation:rule="!has(self.checkpoint) || !has(self.checkpoint.key) || self.checkpoint.key == \"\" || self.checkpoint.key.startsWith((has(self.target.prefix) ? self.target.prefix : \"\") + (has(self.sync) && has(self.sync.destPrefix) ? self.sync.destPrefix : \"\"))",message="checkpoint.key must live under the effective destination prefix (target.prefix + sync.destPrefix)" +type EtcdMirrorSpec struct { + // Mode selects continuous replication (Sync, the default) or a cutover + // drain (Drain). See EtcdMirrorModeDrain for the cutover contract. + // +optional + // +kubebuilder:validation:Enum=Sync;Drain + // +kubebuilder:default=Sync + Mode EtcdMirrorMode `json:"mode,omitempty"` + + // Source is the etcd cluster keys are read from. EtcdMirror never writes + // back to Source; the agent's source-side client is only ever used for + // Get/Watch, never Put/Delete/Txn. + // + // VERSION FLOOR: source etcd must be >= 3.4 (probed via maintenance + // Status() at connect; below the floor the mirror goes Failed with reason + // UnsupportedVersion). >= 3.4.25 / 3.5.8 is the recommended floor: below + // it, watch progress notifications are unreliable and the agent cannot + // trust the watermark machinery that drives lag, the checkpoint, and the + // Drain gate. + Source EtcdMirrorEndpoint `json:"source"` + + // Target is the etcd cluster keys are written into. + // + // SECURITY PREREQUISITE: Target's credential (etcd RBAC user and/or the + // client certificate's role) MUST be range-scoped to the effective + // destination prefix — never a cluster-admin-equivalent credential. The + // agent's client-side rewrite logic is defense against bugs in its own + // code, NOT a security boundary. The grant must also cover the reserved + // checkpoint key (see Checkpoint). Configure via `etcdctl role + // grant-permission --prefix=true readwrite ` before + // pointing an EtcdMirror at the cluster. + // + // The target must run with auto-compaction enabled: forced-resync churn + // and prune passes march an uncompacted target toward its storage quota + // (2GiB by default), which surfaces as TargetQuotaExhausted. + Target EtcdMirrorEndpoint `json:"target"` + + // InitialSync governs genesis behavior: how pre-existing destination keys + // are treated (Mode) and optionally where replication starts + // (StartRevision). + // +optional + InitialSync *EtcdMirrorInitialSyncSpec `json:"initialSync,omitempty"` + + // Sync tunes runtime sync behavior (batching, paging, rate limiting, + // prefix rewrite, timeouts, backoff). + // +optional + Sync EtcdMirrorSyncSpec `json:"sync,omitempty"` + + // Checkpoint configures the reserved checkpoint/fence key on the target. + // +optional + Checkpoint *EtcdMirrorCheckpointSpec `json:"checkpoint,omitempty"` + + // Reconciliation optionally enables a periodic full diff-and-repair pass + // layered on top of the continuous watch-based mirror. Independent of + // this setting, one reconciliation-with-delete pass always runs after any + // forced resync (mark-and-sweep), and as the OverwriteAndPrune genesis + // pass and the Drain verification pass. + // +optional + Reconciliation *EtcdMirrorReconciliationSpec `json:"reconciliation,omitempty"` + + // PodTemplate carries scheduling/affinity/labels/annotations for the + // agent pod, reusing EtcdClusterSpec's PodTemplate shape verbatim. + // +optional + PodTemplate *PodTemplate `json:"podTemplate,omitempty"` + + // Resources are the agent container's compute resources. The agent's + // memory model is bounded by Sync.PageBytes (single in-flight scan page, + // no unbounded read-ahead); size limits accordingly. + // +optional + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + + // Paused, when true, scales the agent Deployment to zero without deleting + // the CR or its checkpoint. The checkpoint lives in the target etcd, so + // resume picks up from the last fenced watermark. NOTE: pausing longer + // than the source's compaction retention guarantees a full forced resync + // on resume — there is no free lunch past the retention window. + // +optional + Paused bool `json:"paused,omitempty"` +} + +// EtcdMirrorInitialSyncSpec governs the genesis scan. +// +// +kubebuilder:validation:XValidation:rule="!(has(self.startRevision) && self.startRevision > 0 && (!has(self.mode) || self.mode == 'RequireEmpty'))",message="initialSync.startRevision requires initialSync.mode Overwrite or OverwriteAndPrune (a seeded target is not empty)" +type EtcdMirrorInitialSyncSpec struct { + // Mode governs pre-existing destination keys at genesis. Defaults to + // RequireEmpty (refuse a non-empty destination prefix). + // +optional + // +kubebuilder:validation:Enum=RequireEmpty;Overwrite;OverwriteAndPrune + // +kubebuilder:default=RequireEmpty + Mode EtcdMirrorInitialSyncMode `json:"mode,omitempty"` + + // StartRevision, when > 0, skips the genesis scan entirely and starts + // watching from StartRevision+1. For fidelity-preserving seeds: restore + // the target from a source snapshot (`etcdutl snapshot restore + // --bump-revision --mark-compacted`), then mirror only the delta. + // Requires Mode Overwrite or OverwriteAndPrune (CEL-enforced). + // +optional + // +kubebuilder:validation:Minimum=0 + StartRevision int64 `json:"startRevision,omitempty"` +} + +// EtcdMirrorEndpoint describes how to reach, authenticate to, and scope one +// side (source or target) of a mirror. Both sides need an identical shape +// (address resolution + prefix + TLS + auth), so one type serves both roles, +// the same way BackupDestination is reused verbatim between EtcdBackup and +// EtcdRestore rather than forked into near-duplicate per-role types. +// +// Exactly one of EndpointList or ServiceRef must be set. An empty +// endpointList ([]) is treated as unset, per Kubernetes list conventions — +// so `endpointList: []` alongside a serviceRef is accepted. +// +// Endpoint scheme and the TLS block must agree (CEL-enforced both ways): +// http:// endpoints with a tls block would silently drop TLS at dial time; +// https:// endpoints without one would dial with undeclared system-roots +// TLS. The agent derives the dial scheme from the presence of the tls block, +// so the declared contract is true by construction. +// +// +kubebuilder:validation:XValidation:rule="(has(self.endpointList) && size(self.endpointList) > 0) != has(self.serviceRef)",message="exactly one of endpointList or serviceRef must be set" +// +kubebuilder:validation:XValidation:rule="!(has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, e.startsWith('http://')))",message="http:// endpoints conflict with a tls block: use https:// endpoints or remove tls" +// +kubebuilder:validation:XValidation:rule="!(!has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, e.startsWith('https://')))",message="https:// endpoints require a tls block (an empty tls block selects server-auth TLS against system trust roots)" +type EtcdMirrorEndpoint struct { + // EndpointList is a raw set of etcd client-URL host:port (or + // scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379". + // This is the ONLY supported mechanism for a cluster external to this + // Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB); + // there is deliberately no tunnel/port-forward mode — terminate any + // tunnel upstream and hand EtcdMirror the resulting stable endpoint(s). + // + // Prefer listing per-member endpoints over a single load-balancer VIP: a + // TCP-health-checked VIP cannot see etcd quorum, and the client's own + // balancer handles per-member failover. + // + // IP-LITERAL ENDPOINTS: Go's TLS stack requires an IP SAN (not a DNS SAN) + // to verify a bare-IP endpoint. Either set EtcdMirrorTLS.ServerName to a + // hostname present as a DNS SAN on the certificate, or ensure the + // certificate carries a matching IP SAN. Fix the SAN/ServerName mismatch; + // do not reach for InsecureSkipVerify. + // +optional + EndpointList []string `json:"endpointList,omitempty"` + + // ServiceRef points at a Kubernetes Service in this cluster whose DNS + // name resolves the etcd client endpoint(s), for the same-cluster case. + // Namespace defaults to the EtcdMirror's own namespace when empty. + // +optional + ServiceRef *EtcdMirrorServiceRef `json:"serviceRef,omitempty"` + + // Prefix is the etcd key prefix on THIS side. On Source, only keys under + // this prefix are synced; empty means the whole keyspace. On Target, this + // is the prefix under which mirrored keys land (see EtcdMirrorSyncSpec's + // rewrite formula). Immutable after creation. + // +optional + Prefix string `json:"prefix,omitempty"` + + // TLS configures the agent's client TLS for THIS side. Nil means the + // agent dials this side in cleartext (and https:// endpoints are + // CEL-rejected). An empty block means server-auth TLS verified against + // the system trust roots. + // +optional + TLS *EtcdMirrorTLS `json:"tls,omitempty"` + + // Auth configures etcd username/password (RBAC) auth for THIS side. + // +optional + Auth *EtcdMirrorAuth `json:"auth,omitempty"` +} + +// EtcdMirrorServiceRef points at a Service and the client port on it to dial. +type EtcdMirrorServiceRef struct { + // Name is the Service name. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // Namespace defaults to the EtcdMirror's own namespace when empty. + // +optional + Namespace string `json:"namespace,omitempty"` + + // Port is the Service port name or number exposing etcd's client API. + // Defaults to "client" when empty, matching PodMonitorSpec.Port's + // convention elsewhere in this API group. + // +optional + Port string `json:"port,omitempty"` +} + +// EtcdMirrorTLS configures the mirror agent's client TLS for one side of a +// mirror. Plain secretRef, not a reuse of EtcdClusterTLS/TLSSurface — the +// agent is always a client to clusters it does not own, so issuer-selection +// machinery doesn't apply. +// +// ROTATION CONTRACT: the agent re-reads TLS material from the mounted Secret +// on every handshake (transport.TLSInfo file paths, not a one-shot +// tls.Config); certificate rotation requires no pod restart. +// +// +kubebuilder:validation:XValidation:rule="!self.insecureSkipVerify || self.insecureSkipVerifyAcknowledgeRisk",message="insecureSkipVerify requires insecureSkipVerifyAcknowledgeRisk to also be true" +// +kubebuilder:validation:XValidation:rule="!has(self.secretRef) || (has(self.secretRef.name) && size(self.secretRef.name) > 0)",message="secretRef.name must be non-empty when secretRef is set" +type EtcdMirrorTLS struct { + // SecretRef names a Secret (in the EtcdMirror's namespace) holding this + // side's TLS material in the standard kubernetes.io/tls-compatible shape: + // - ca.crt: PEM CA bundle used to verify the peer's server certificate + // (unless CABundleRef overrides it, or InsecureSkipVerify is true). + // - tls.crt / tls.key: PEM client certificate + key, for mTLS. + // Optional — omit both for server-auth-only TLS. + // Nil means no client identity and verification against the system trust + // roots (the etcdctl default). Note a source running with + // --client-cert-auth (the RKE1 default) rejects certless clients at the + // handshake regardless of etcd RBAC auth; server-auth-only + Auth is not + // viable against such a source. + // +optional + SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"` + + // CABundleRef optionally sources the trust anchors from a separate + // Secret or ConfigMap key, decoupling trust from the identity Secret + // (Gateway API caCertificateRefs precedent). Takes precedence over + // SecretRef's ca.crt. + // +optional + CABundleRef *EtcdMirrorCABundleRef `json:"caBundleRef,omitempty"` + + // InsecureSkipVerify disables server certificate verification. Strongly + // discouraged, especially for a source reached over the public internet. + // Requires InsecureSkipVerifyAcknowledgeRisk to also be set true + // (CEL-enforced). The controller additionally emits a standing Warning + // event whenever this is true. + // +optional + // +kubebuilder:default=false + InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"` + + // InsecureSkipVerifyAcknowledgeRisk must independently be set true + // whenever InsecureSkipVerify is true (CEL-enforced companion field). Its + // only purpose is to require a deliberate, separate, reviewable line in + // the manifest diff before disabling TLS verification. + // +optional + // +kubebuilder:default=false + InsecureSkipVerifyAcknowledgeRisk bool `json:"insecureSkipVerifyAcknowledgeRisk,omitempty"` + + // ServerName overrides the TLS ServerName (SNI) used for verification, + // for cases where the dialed address doesn't match a SAN on the + // certificate (e.g. dialing an NLB IP directly). Applies to EVERY + // endpoint in the list, so mixing endpoints with different certificates + // behind one ServerName will fail verification on the mismatched ones. + // +optional + ServerName string `json:"serverName,omitempty"` +} + +// EtcdMirrorCABundleRef points at one key of a Secret or ConfigMap holding a +// PEM CA bundle. +type EtcdMirrorCABundleRef struct { + // Kind is Secret or ConfigMap. Defaults to ConfigMap. + // +optional + // +kubebuilder:validation:Enum=Secret;ConfigMap + // +kubebuilder:default=ConfigMap + Kind string `json:"kind,omitempty"` + + // Name of the Secret or ConfigMap, in the EtcdMirror's namespace. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // Key within the object. Defaults to "ca.crt". + // +optional + Key string `json:"key,omitempty"` +} + +// EtcdMirrorAuth configures etcd RBAC username/password auth for one side. +// +// +kubebuilder:validation:XValidation:rule="has(self.secretRef.name) && size(self.secretRef.name) > 0",message="secretRef.name is required" +type EtcdMirrorAuth struct { + // SecretRef names a Secret holding "username" and "password" keys. If + // this whole Auth block is nil, the agent does not call etcd's + // Authenticate() at all; the pinned v3 client transparently re-auths on + // token expiry. PRECEDENCE: when both a client certificate and Auth are + // supplied, etcd uses the token identity, not the certificate CN — the + // Auth user must hold the range-scoped role. + SecretRef corev1.LocalObjectReference `json:"secretRef"` +} + +// EtcdMirrorSyncSpec tunes the mirror's runtime sync behavior. +// +// KEY REWRITE — one formula, no other composition: +// +// key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix) +// +// (anchored strip-and-reprefix; never a substring replace). +// +// BATCHING INVARIANT: target Txns flush ONLY at source-revision boundaries — +// a source revision's events are never split across Txns, whole revisions +// are coalesced up to the MaxTxnOps/TxnFlushBytes watermarks, and one op +// slot in MaxTxnOps is always reserved for the checkpoint write that rides +// in the same Txn. A single source revision larger than MaxTxnOps is applied +// as one oversized Txn (provision the target's --max-txn-ops accordingly) +// with the checkpoint held until it lands. +// +// RETENTION PREREQUISITE: the source's compaction retention window must +// exceed the worst-case initial scan + throttled drain time, approximately +// sourceKeyCount / min(effective scan rate, MaxOpsPerSecond). If it does +// not, genesis (and every forced resync) loses the race with compaction and +// the mirror livelocks (surfaced via the resync-loop detector). +type EtcdMirrorSyncSpec struct { + // DestPrefix is the middle term of the rewrite formula above. Default "" + // means the source prefix is stripped and key remainders land directly + // under target.prefix. Immutable after creation. + // +optional + DestPrefix string `json:"destPrefix,omitempty"` + + // ExcludePrefixes lists source key prefixes (full source-side keys, e.g. + // "/registry/events/") skipped entirely: not scanned, not watched, not + // counted, not pruned. Use to drop high-churn low-value ranges and cut + // WAN cost, or to skip lease-backed ranges that don't survive mirroring. + // Nested/duplicate entries are normalized by the agent (an entry covered + // by another is dropped). Immutable after creation (it defines the + // mirrored range): removing an exclusion would require a backfill scan + // the checkpoint-resume path never runs, and adding one would strand + // already-mirrored keys as permanent orphans — change it via + // delete-and-recreate with an appropriate initialSync.mode instead. + // +optional + // +kubebuilder:validation:MaxItems=64 + ExcludePrefixes []string `json:"excludePrefixes,omitempty"` + + // MaxTxnOps bounds how many operations the agent batches into a single + // target Txn, including the reserved checkpoint-write slot. Must not + // exceed the target's --max-txn-ops (etcd default 128). Defaults to 128. + // +optional + // +kubebuilder:validation:Minimum=2 + MaxTxnOps int32 `json:"maxTxnOps,omitempty"` + + // TxnFlushBytes is the byte watermark at which a batch is flushed (at the + // next source-revision boundary). Keep well under etcd's request size + // limits: a Txn over ~1.5MiB is rejected by the server and one over 2MiB + // by the client send cap — both classified permanent errors, not + // throttling. Defaults to 1Mi. + // +optional + TxnFlushBytes *resource.Quantity `json:"txnFlushBytes,omitempty"` + + // PageKeyLimit bounds keys per source scan page during InitialSync and + // reconciliation. The scan is pull-based, one page in flight — no + // read-ahead — so this and PageBytes bound agent memory. Defaults to 512. + // +optional + // +kubebuilder:validation:Minimum=1 + PageKeyLimit int32 `json:"pageKeyLimit,omitempty"` + + // PageBytes bounds bytes per source scan page. Defaults to 1Mi. + // +optional + PageBytes *resource.Quantity `json:"pageBytes,omitempty"` + + // WatchBufferBytes bounds the memory used to buffer watch events + // observed from R0 while the genesis scan runs (the reflector replay + // buffer). On overflow the agent cancels the source watch and restarts + // the scan from a fresh R0 (see the InitialSyncCompactionRaced event) — + // a bounded retry instead of unbounded growth when source churn outruns + // scan+apply throughput. Defaults to 16Mi (must stay in lockstep with + // pkg/mirroragent's DefaultWatchBufferBytes). + // +optional + WatchBufferBytes *resource.Quantity `json:"watchBufferBytes,omitempty"` + + // MaxOpsPerSecond rate-limits the agent's target write rate (a token + // bucket over puts+deletes/sec), applied to both the genesis scan and + // watch-driven applies. Zero (default) means unlimited. Mind the + // retention prerequisite above when throttling. + // +optional + // +kubebuilder:validation:Minimum=0 + MaxOpsPerSecond int32 `json:"maxOpsPerSecond,omitempty"` + + // RequestTimeout is the per-RPC context deadline applied to every unary + // call on both sides (watches excluded — they are long-lived by design + // and covered by progress-notification liveness instead). Without it a + // blackholed call through an NLB never errors and backoff never engages. + // Defaults to 30s. + // +optional + RequestTimeout *metav1.Duration `json:"requestTimeout,omitempty"` + + // DialTimeout bounds establishing the initial client connection to each + // side. Defaults to 10s. + // +optional + DialTimeout *metav1.Duration `json:"dialTimeout,omitempty"` + + // ReconnectBackoff bounds the retry/backoff loop wrapping connection-class + // errors. Throttling-class errors (target rate rejection) use a more + // conservative curve derived from the same bounds; quota exhaustion + // (TargetQuotaExhausted) and permanent errors are never retried through + // this loop. Defaults to exponential backoff from 1s to 30s. + // +optional + ReconnectBackoff *EtcdMirrorBackoffSpec `json:"reconnectBackoff,omitempty"` +} + +type EtcdMirrorBackoffSpec struct { + // +optional + InitialDelay *metav1.Duration `json:"initialDelay,omitempty"` + // +optional + MaxDelay *metav1.Duration `json:"maxDelay,omitempty"` +} + +// EtcdMirrorCheckpointSpec configures the reserved checkpoint/fence key the +// agent maintains IN THE TARGET etcd. The checkpoint (the source-revision +// watermark plus {linkUID, epoch, role}) is written in the SAME Txn as every +// applied batch and fenced with a mod-revision compare on EVERY write path +// (applies, reconciliation repairs, prune deletes), so two agents can never +// interleave writes and a straggler apply after cutover fails loudly. The +// key is excluded by exact match from scans, counts, prune passes, and the +// RequireEmpty check; the target RBAC grant must cover it; CR deletion +// removes it via a delete-one-key finalizer. +type EtcdMirrorCheckpointSpec struct { + // Key overrides the reserved checkpoint key. Defaults to the effective + // destination prefix + "\x00etcdmirror-checkpoint" — the \x00 byte after + // the prefix cannot collide with any real key under it. MUST live under + // the effective destination prefix (target.prefix + sync.destPrefix, + // CEL-enforced): the range-scoped target credential covers it, and the + // exact-match exclusion from scans/counts/prune only works inside the + // mirrored range. Immutable after creation. + // +optional + Key string `json:"key,omitempty"` +} + +// EtcdMirrorReconciliationSpec configures the periodic full reconciliation +// pass. The same engine also runs unconditionally (regardless of Enabled or +// DeleteOrphans) as the post-forced-resync mark-and-sweep, the +// OverwriteAndPrune genesis pass, and the Drain verification pass. +type EtcdMirrorReconciliationSpec struct { + // Enabled toggles the PERIODIC pass. Defaults to false: it is a full + // diff of the prefix contents on both sides (O(keyspace)), so it is + // opt-in. + // +optional + Enabled bool `json:"enabled,omitempty"` + + // Interval between periodic passes. Defaults to 1h when Enabled. + // +optional + Interval *metav1.Duration `json:"interval,omitempty"` + + // DeleteOrphans, when true, allows the PERIODIC pass to delete target + // keys under the destination prefix that have no corresponding source + // key. Defaults to false. (Forced-resync sweeps and OverwriteAndPrune + // always delete orphans; this knob only governs the periodic pass.) + // +optional + DeleteOrphans bool `json:"deleteOrphans,omitempty"` +} + +// EtcdMirrorPhase is a high-level summary of an EtcdMirror's lifecycle. +// Unlike BackupPhase/RestorePhase, most phases here are NOT terminal — a +// healthy mirror spends its life in Syncing; there is no "Completed" state. +type EtcdMirrorPhase string + +const ( + // EtcdMirrorPhasePending means the EtcdMirror has been accepted but the + // agent workload has not been created yet. + EtcdMirrorPhasePending EtcdMirrorPhase = "Pending" + // EtcdMirrorPhaseConnecting means the agent pod is running, establishing + // client connections to both sides and probing versions/cluster IDs. + EtcdMirrorPhaseConnecting EtcdMirrorPhase = "Connecting" + // EtcdMirrorPhaseInitialSync means the agent is running the genesis scan: + // an UNPINNED chunked scan with the watch already open from the revision + // observed before the scan started, buffered events replayed over the + // scanned base (reflector pattern). Because pages read at the current + // revision, mid-scan compaction cannot fail the scan. Also entered during + // a forced resync (then with condition Compacted=True/Reason=ForcedResync). + EtcdMirrorPhaseInitialSync EtcdMirrorPhase = "InitialSync" + // EtcdMirrorPhaseSyncing is the steady state: watching and applying live + // changes, watermark advancing via progress notifications. A completed + // drain also reports Syncing (there is no Drained phase — a finished + // cutover is not page-worthy); the CutoverReady condition carries the + // drain outcome. + EtcdMirrorPhaseSyncing EtcdMirrorPhase = "Syncing" + // EtcdMirrorPhaseDegraded means the agent is in a retry/backoff loop + // (connection or throttling class) and is expected to self-heal. Forced + // resyncs are NOT Degraded; they report as InitialSync + Compacted=True. + EtcdMirrorPhaseDegraded EtcdMirrorPhase = "Degraded" + // EtcdMirrorPhasePaused means spec.paused is true; the agent Deployment + // is scaled to zero. The checkpoint is retained in the target. + EtcdMirrorPhasePaused EtcdMirrorPhase = "Paused" + // EtcdMirrorPhaseFailed means a terminal, non-recoverable error requiring + // operator intervention: EmptyTargetViolation at genesis, source below + // the 3.4 version floor (UnsupportedVersion), a permanent-class write + // error (oversized revision vs target limits), malformed cert material, + // or unresolvable spec misconfiguration. + EtcdMirrorPhaseFailed EtcdMirrorPhase = "Failed" +) + +// Condition types reported on EtcdMirror status. +// +// PAGING ALGEBRA (for alert authors): page on Available=False sustained for +// your tolerance window UNLESS Compacted=True AND progress fields are +// advancing (a forced resync healing itself); TargetQuotaExhausted and +// ResyncLoopDetected page immediately — neither self-heals. +const ( + // EtcdMirrorConditionAvailable is True only when the agent pod is + // running, in the Syncing phase, AND the checkpoint watermark is + // advancing (via applies or watch progress notifications) within the + // staleness threshold. Watermark-derived, not apply-derived: an idle + // prefix on a live watch stays Available; a wedged loop that stops + // confirming progress does not. Exception: after a completed, verified + // drain the watermark never advances again by design, and Available + // stays True with reason DrainComplete (phase remains Syncing; + // CutoverReady carries the outcome — a finished cutover must not page). + EtcdMirrorConditionAvailable = "Available" + // EtcdMirrorConditionSourceReachable is True when the agent's last + // attempt to reach Source succeeded. Split from TargetReachable because + // in the primary use case (source over the public internet) source + // reachability is the most likely persistent failure mode. + EtcdMirrorConditionSourceReachable = "SourceReachable" + // EtcdMirrorConditionTargetReachable is the target-side analogue. + EtcdMirrorConditionTargetReachable = "TargetReachable" + // EtcdMirrorConditionTargetThrottled is True while the agent is backing + // off from throttling-class errors on Target (rate rejection / + // ErrTooManyRequests). Distinct from TargetReachable=False ("up but + // rejecting my write rate" is actionable differently) and from + // TargetQuotaExhausted (backoff cannot heal a full quota). + EtcdMirrorConditionTargetThrottled = "TargetThrottled" + // EtcdMirrorConditionTargetQuotaExhausted is True when a target write + // failed with etcd's NOSPACE (rpctypes.ErrNoSpace). Permanent until an + // operator compacts/defrags/disarms the target; the agent stops writing + // rather than burning backoff against a full quota. Detected from the + // typed write-path error, never AlarmList (which needs root). + EtcdMirrorConditionTargetQuotaExhausted = "TargetQuotaExhausted" + // EtcdMirrorConditionInitialSyncComplete is True once the genesis scan + // has completed against the currently-checkpointed cluster identities. + // Durable across forced resyncs (Compacted covers those); reset to False + // when the checkpoint is invalidated by a mismatch of EITHER bound + // cluster ID (source or target), which also re-arms the RequireEmpty + // check. + EtcdMirrorConditionInitialSyncComplete = "InitialSyncComplete" + // EtcdMirrorConditionCompacted is True (Reason ForcedResync) while the + // agent heals from source compaction outrunning the watch (restart or + // pause longer than retention; a WatchResponse with CompactRevision != + // 0). Mid-scan compaction is NOT in this class — the unpinned scan is + // immune by construction. Every forced resync ends with a mandatory + // mark-and-sweep prune. Reverts to False when steady-state resumes. + EtcdMirrorConditionCompacted = "Compacted" + // EtcdMirrorConditionResyncLoopDetected is True when N consecutive forced + // resyncs completed without reaching steady state — the livelock + // signature of source retention < scan+drain time. Does not self-heal: + // raise retention, raise MaxOpsPerSecond, or shrink the prefix. + EtcdMirrorConditionResyncLoopDetected = "ResyncLoopDetected" + // EtcdMirrorConditionReplicationLagExceeded is True when the checkpoint + // watermark has stayed more than a threshold behind the source's + // current revision for a sustained duration. Both terms come from the + // same watch/progress machinery (never from comparing the two live + // status fields, which snapshot at different instants). Threshold and + // duration are controller-internal constants in v1. + EtcdMirrorConditionReplicationLagExceeded = "ReplicationLagExceeded" + // EtcdMirrorConditionDriftDetected is True when the last reconciliation + // pass found orphaned/missing keys. Carries counts in Message. Sticky + // until the next pass reports clean. + EtcdMirrorConditionDriftDetected = "DriftDetected" + // EtcdMirrorConditionEmptyTargetViolation is True (terminal, Phase -> + // Failed) when initialSync.mode is RequireEmpty and the destination + // prefix was non-empty at genesis. The condition Message embeds the + // exact `etcdctl del` command for the offending range. The reserved + // checkpoint key is excluded by exact match. + EtcdMirrorConditionEmptyTargetViolation = "EmptyTargetViolation" + // EtcdMirrorConditionCutoverReady is True when spec.mode is Drain, the + // watermark has reached status.cutover.drainTargetRevision, and the + // verification pass succeeded. From then the fence key's role is + // Primary and any straggler mirror apply fails its compare. Gate + // promotion on `kubectl wait --for=condition=CutoverReady`. + EtcdMirrorConditionCutoverReady = "CutoverReady" + // EtcdMirrorConditionInvariantsHeld is the composed verification verdict: + // True when ReplicationLagExceeded is False, the per-side key counts + // (status.sourceKeyCount/targetKeyCount, reserved key excluded) are + // equal, DriftDetected is False, and the pass that produced the counts + // is fresh. Freshness: with the periodic reconciliation pass enabled, + // within 2x spec.reconciliation.interval; with it disabled the counts + // only refresh on mandatory passes (forced-resync sweeps, + // OverwriteAndPrune genesis, drain verification), so the condition is + // Unknown/stale-reasoned once the last such pass ages out — enable + // reconciliation to make this a continuous signal. It means + // "verification invariants hold", never "safe to cut over" — cutover is + // gated on CutoverReady, which additionally requires spec.mode Drain and + // a reached drainTargetRevision. + EtcdMirrorConditionInvariantsHeld = "InvariantsHeld" + // EtcdMirrorConditionLearnerEndpoint is True when the maintenance + // Status() probe reports IsLearner=true for a configured endpoint. + // Non-blocking (learners self-heal and the balancer routes around them), + // but a learner pick can serve stale reads mid-catch-up. + EtcdMirrorConditionLearnerEndpoint = "LearnerEndpoint" + // EtcdMirrorConditionPrefixConflict is controller-set: another + // EtcdMirror targets an overlapping effective destination range on the + // same target cluster. Declared now so the name is API contract before + // any setter exists. Independent of the controller check, the agent + // itself refuses (permanently, Phase=Failed) to prune a reserved fence + // key owned by a different link, so an undetected overlap stops loudly + // instead of destroying the sibling mirror's fence and data. + EtcdMirrorConditionPrefixConflict = "PrefixConflict" + // EtcdMirrorConditionDirectionConflict is controller-set: this + // mirror's bound source/target cluster IDs are the inverse of another + // EtcdMirror's — two CRs forming a two-way loop, caught by cluster-ID + // binding even when a respelled endpoint string would fool a spec + // comparison. + EtcdMirrorConditionDirectionConflict = "DirectionConflict" +) + +// Condition reasons and Event reasons that are part of the API contract. +const ( + // EtcdMirrorReasonForcedResync is the Compacted condition's reason while + // a forced resync is in flight. + EtcdMirrorReasonForcedResync = "ForcedResync" + // EtcdMirrorReasonUnsupportedVersion is the Failed-phase reason when the + // source is below the 3.4 floor. + EtcdMirrorReasonUnsupportedVersion = "UnsupportedVersion" + + // EtcdMirrorReasonCompacted / EtcdMirrorReasonClusterIDMismatch name WHY + // a forced resync was required (mirroring pkg/mirroragent's ResyncReason + // values). Surfaces: the Compacted condition's MESSAGE and the + // forced-resync events' messages (and, later, the forced-resync metric's + // reason label). The Compacted condition's Reason is always ForcedResync + // while a resync is in flight — alert on that, not on these — and + // forcedResyncCount is a plain counter with no per-reason breakdown. The + // ClusterIDMismatch message additionally names the mismatched side. + EtcdMirrorReasonCompacted = "Compacted" + EtcdMirrorReasonClusterIDMismatch = "ClusterIDMismatch" + // EtcdMirrorReasonCheckpointInvalid is a Failed-phase reason: the stored + // checkpoint was corrupt or of an unknown wire version. PERMANENT — the + // agent fails closed and never auto-resyncs; the operator must inspect + // and delete the reserved key to recover. + EtcdMirrorReasonCheckpointInvalid = "CheckpointInvalid" + + // EtcdMirrorEventForcedResyncStarted / Completed bracket every forced + // resync (Warning/Normal respectively). + EtcdMirrorEventForcedResyncStarted = "ForcedResyncStarted" + EtcdMirrorEventForcedResyncCompleted = "ForcedResyncCompleted" + // EtcdMirrorEventCheckpointInvalidated is emitted when the checkpoint is + // discarded because a bound cluster ID (source or target) no longer + // matches, forcing genesis and re-arming the RequireEmpty check. + EtcdMirrorEventCheckpointInvalidated = "CheckpointInvalidated" + // EtcdMirrorEventInitialSyncCompactionRaced marks an InitialSync attempt + // aborted and restarted from a fresh R0. Two causes, named in the event + // message: WatchBufferOverflow (the replay buffer exceeded + // sync.watchBufferBytes before the base scan completed — a memory-bound + // retry, NOT a compaction race) and WatchCompactedMidScan (a watch + // reconnect landed below the source compact revision — the rare genuine + // race). One event name so operators can alert on scan restarts; the + // cause string prevents conflating "buffer too small for churn" with + // "compaction won a race the design eliminates". Repeated occurrences + // count toward ResyncLoopDetected. + EtcdMirrorEventInitialSyncCompactionRaced = "InitialSyncCompactionRaced" + // EtcdMirrorEventCertificateExpiringSoon warns that a referenced TLS + // certificate (client leaf or CA) expires within the controller's + // lead window; the agent's expiry gauge is the metric counterpart. + EtcdMirrorEventCertificateExpiringSoon = "CertificateExpiringSoon" + // EtcdMirrorEventInsecureSkipVerifyEnabled is the standing Warning + // promised by EtcdMirrorTLS.InsecureSkipVerify's contract. + EtcdMirrorEventInsecureSkipVerifyEnabled = "InsecureSkipVerifyEnabled" +) + +// EtcdMirrorStatus defines the observed state of an EtcdMirror. Progress +// fields are synced periodically (not per-op), so they are a coarse, +// point-in-time mirror of the agent's authoritative checkpoint in the target +// etcd, useful for kubectl/dashboards, not an audit log. +// +// CUTOVER GATE CONTRACT: never compute "caught up" by comparing +// SourceRevision to LastAppliedRevision — they snapshot at different +// instants, and SourceRevision advances on out-of-prefix writes (revisions +// are cluster-global). The manual gate is: quiesce source writers, read the +// source's current revision R yourself (`etcdctl endpoint status`), then +// poll until LastAppliedRevision >= R. The in-CR gate is spec.mode=Drain + +// the CutoverReady condition. Relax target RBAC only after the mirror is +// paused or deleted. +type EtcdMirrorStatus struct { + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // +optional + Phase EtcdMirrorPhase `json:"phase,omitempty"` + + // LastAppliedRevision is the checkpoint watermark: the source revision + // through which the target is caught up, advanced by applies AND by + // watch progress notifications on idle prefixes. The fenced checkpoint + // key in the target etcd is the authoritative copy; this mirrors it. + // +optional + LastAppliedRevision int64 `json:"lastAppliedRevision,omitempty"` + + // SourceRevision is the source cluster's revision as of the last status + // sync. Cluster-global: it advances on writes outside the mirrored + // prefix, so SourceRevision - LastAppliedRevision OVERSTATES lag for + // prefix-scoped mirrors. See the cutover gate contract above. + // +optional + SourceRevision int64 `json:"sourceRevision,omitempty"` + + // SourceClusterID and TargetClusterID are the cluster IDs both bound + // into the checkpoint. Either changing across reconciles is the visible + // symptom of an endpoint now pointing at a different cluster than the + // checkpoint was taken against (CheckpointInvalidated event, forced + // genesis, RequireEmpty re-armed). + // +optional + SourceClusterID string `json:"sourceClusterID,omitempty"` + // +optional + TargetClusterID string `json:"targetClusterID,omitempty"` + + // SourceVersion and TargetVersion are the etcd server versions from the + // maintenance Status() probe at connect. + // +optional + SourceVersion string `json:"sourceVersion,omitempty"` + // +optional + TargetVersion string `json:"targetVersion,omitempty"` + + // InitialSyncKeyCount is the number of keys applied so far by the + // current/last genesis scan. Live-updating during InitialSync (each + // status sync), so InitialSyncKeyCount/InitialSyncTotalKeyCount is a + // progress fraction. + // +optional + InitialSyncKeyCount int64 `json:"initialSyncKeyCount,omitempty"` + // InitialSyncTotalKeyCount is the denominator: the source-side key count + // under the prefix observed at scan start (first page's RangeResponse + // count). + // +optional + InitialSyncTotalKeyCount int64 `json:"initialSyncTotalKeyCount,omitempty"` + // +optional + InitialSyncStartTime *metav1.Time `json:"initialSyncStartTime,omitempty"` + // +optional + InitialSyncCompletionTime *metav1.Time `json:"initialSyncCompletionTime,omitempty"` + + // LeaseBackedKeyCount is the number of mirrored keys whose source copy is + // lease-backed (kv.Lease != 0). Mirrored copies are NOT lease-backed — + // leases are stripped (see Fidelity Caveats in docs/etcdmirror.md) — so a + // nonzero count means the cutover runbook's purge/re-lease step applies. + // +optional + LeaseBackedKeyCount int64 `json:"leaseBackedKeyCount,omitempty"` + + // ForcedResyncCount counts forced resyncs (compaction outran the watch, + // checkpoint invalidated by a cluster-ID mismatch, or checkpoint + // corrupt/unknown-version). Monotonic, never reset. + // +optional + ForcedResyncCount int32 `json:"forcedResyncCount,omitempty"` + + // ScanRestartCount counts genesis-scan attempts aborted and restarted + // from a fresh R0 (watch-buffer overflow or a mid-scan watch compaction; + // see the InitialSyncCompactionRaced event). Monotonic, never reset. + // +optional + ScanRestartCount int64 `json:"scanRestartCount,omitempty"` + + // LastReconciliationTime and LastReconciliationDrift record the most + // recent reconciliation pass (periodic or mandatory). + // +optional + LastReconciliationTime *metav1.Time `json:"lastReconciliationTime,omitempty"` + // +optional + LastReconciliationDrift *EtcdMirrorDriftInfo `json:"lastReconciliationDrift,omitempty"` + + // SourceKeyCount and TargetKeyCount are the per-side key counts from the + // most recent diff/verification pass (reserved checkpoint key and + // excluded prefixes not counted; drain-verification source reads pinned + // at the drained revision with a compacted-fallback re-read). Populated + // by every pass that runs regardless of spec.reconciliation.enabled — + // the mandatory mark-and-sweep after any forced resync, the + // OverwriteAndPrune genesis pass, and the drain verification — plus the + // periodic pass when it is enabled. NOT refreshed on every status sync: + // a healthy RequireEmpty mirror that never forces a resync only gets + // counts from an enabled periodic pass. This is the equality signal + // InvariantsHeld reads; status.cutover's copies remain the frozen + // drain-time snapshot. + // +optional + SourceKeyCount int64 `json:"sourceKeyCount,omitempty"` + // +optional + TargetKeyCount int64 `json:"targetKeyCount,omitempty"` + + // LastStatusSyncTime is when status was last refreshed from the agent, so + // staleness of everything above is directly observable. + // +optional + LastStatusSyncTime *metav1.Time `json:"lastStatusSyncTime,omitempty"` + + // LastProgressTime is when the watermark last advanced (apply or watch + // progress notification). Distinct from LastStatusSyncTime: status can + // keep syncing from a wedged loop; this field is what Available and + // ReplicationLagExceeded are derived from. + // +optional + LastProgressTime *metav1.Time `json:"lastProgressTime,omitempty"` + + // Cutover is populated while spec.mode is Drain. + // +optional + Cutover *EtcdMirrorCutoverStatus `json:"cutover,omitempty"` + + // AgentPod is the name of the current agent pod (the sole pod of the + // size-1 Deployment), for convenient kubectl logs/exec. + // +optional + AgentPod string `json:"agentPod,omitempty"` + + // +optional + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// EtcdMirrorCutoverStatus tracks a Drain-mode cutover. +type EtcdMirrorCutoverStatus struct { + // DrainTargetRevision is the source revision observed when Drain was + // requested — the revision the watermark must reach. + // +optional + DrainTargetRevision int64 `json:"drainTargetRevision,omitempty"` + // DrainedRevision is the watermark at which the drain completed. + // +optional + DrainedRevision int64 `json:"drainedRevision,omitempty"` + // VerifiedTime is when the post-drain verification pass succeeded. + // +optional + VerifiedTime *metav1.Time `json:"verifiedTime,omitempty"` + // SourceKeyCount and TargetKeyCount are the per-side key counts from the + // verification pass (source read pinned at the drained revision; + // reserved checkpoint key excluded). + // +optional + SourceKeyCount int64 `json:"sourceKeyCount,omitempty"` + // +optional + TargetKeyCount int64 `json:"targetKeyCount,omitempty"` + // LeasedKeyCount is LeaseBackedKeyCount frozen at drain completion, for + // the runbook's purge/re-lease step. + // +optional + LeasedKeyCount int64 `json:"leasedKeyCount,omitempty"` +} + +type EtcdMirrorDriftInfo struct { + // MissingKeys were present on the source but absent on the target. + MissingKeys int64 `json:"missingKeys,omitempty"` + // DivergentKeys were present on both sides with different values — + // distinct from MissingKeys so "a resync dropped keys" is never + // conflated with "a blind window went stale". + DivergentKeys int64 `json:"divergentKeys,omitempty"` + // OrphanKeys were present on the target with no source counterpart. + OrphanKeys int64 `json:"orphanKeys,omitempty"` + // Repaired is true when the pass wrote fixes rather than only reporting. + Repaired bool `json:"repaired,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Available",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].status` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Revision",type=integer,JSONPath=`.status.lastAppliedRevision` +// +kubebuilder:printcolumn:name="Source-Rev",type=integer,JSONPath=`.status.sourceRevision` +// +kubebuilder:printcolumn:name="Last-Progress",type=date,JSONPath=`.status.lastProgressTime` +// +kubebuilder:printcolumn:name="Source",type=string,priority=1,JSONPath=`.spec.source.prefix` +// +kubebuilder:printcolumn:name="Target",type=string,priority=1,JSONPath=`.spec.target.prefix` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// EtcdMirror is the Schema for the etcdmirrors API. It describes a +// continuous, one-way key-range sync from a source etcd cluster to a target +// etcd cluster, run as a single supervised stateless pod (a size-1 +// Deployment; progress lives in a fenced checkpoint key in the target etcd, +// not on a volume). +// +// It is a byte-copy of keys and values, not a replica: revisions, versions, +// and create/mod ordering are target-assigned, and leases are stripped. See +// Fidelity Caveats in docs/etcdmirror.md before depending on anything but +// key/value content. +// +// Two-way sync: never — etcd revisions are cluster-local and there is no +// per-key provenance channel, so bidirectional sync is structurally +// inexpressible. Reversal for cutover/failback: yes — delete the CR and +// create a new one with swapped endpoints, initialSync.mode +// OverwriteAndPrune, only after the forward mirror reported CutoverReady. +// See docs/etcdmirror.md. +type EtcdMirror struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec EtcdMirrorSpec `json:"spec,omitempty"` + Status EtcdMirrorStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// EtcdMirrorList contains a list of EtcdMirror. +type EtcdMirrorList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EtcdMirror `json:"items"` +} diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go index 9a178715..54ef002f 100644 --- a/api/v1alpha1/groupversion_info.go +++ b/api/v1alpha1/groupversion_info.go @@ -40,6 +40,8 @@ func addKnownTypes(s *runtime.Scheme) error { s.AddKnownTypes(GroupVersion, &EtcdCluster{}, &EtcdClusterList{}, + &EtcdMirror{}, + &EtcdMirrorList{}, ) metav1.AddToGroupVersion(s, GroupVersion) return nil diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9db6ca12..0f6a17af 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha1 import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" netx "net" ) @@ -200,6 +200,425 @@ func (in *EtcdClusterStatus) DeepCopy() *EtcdClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirror) DeepCopyInto(out *EtcdMirror) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirror. +func (in *EtcdMirror) DeepCopy() *EtcdMirror { + if in == nil { + return nil + } + out := new(EtcdMirror) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EtcdMirror) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorAuth) DeepCopyInto(out *EtcdMirrorAuth) { + *out = *in + out.SecretRef = in.SecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorAuth. +func (in *EtcdMirrorAuth) DeepCopy() *EtcdMirrorAuth { + if in == nil { + return nil + } + out := new(EtcdMirrorAuth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorBackoffSpec) DeepCopyInto(out *EtcdMirrorBackoffSpec) { + *out = *in + if in.InitialDelay != nil { + in, out := &in.InitialDelay, &out.InitialDelay + *out = new(metav1.Duration) + **out = **in + } + if in.MaxDelay != nil { + in, out := &in.MaxDelay, &out.MaxDelay + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorBackoffSpec. +func (in *EtcdMirrorBackoffSpec) DeepCopy() *EtcdMirrorBackoffSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorBackoffSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorCABundleRef) DeepCopyInto(out *EtcdMirrorCABundleRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorCABundleRef. +func (in *EtcdMirrorCABundleRef) DeepCopy() *EtcdMirrorCABundleRef { + if in == nil { + return nil + } + out := new(EtcdMirrorCABundleRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorCheckpointSpec) DeepCopyInto(out *EtcdMirrorCheckpointSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorCheckpointSpec. +func (in *EtcdMirrorCheckpointSpec) DeepCopy() *EtcdMirrorCheckpointSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorCheckpointSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorCutoverStatus) DeepCopyInto(out *EtcdMirrorCutoverStatus) { + *out = *in + if in.VerifiedTime != nil { + in, out := &in.VerifiedTime, &out.VerifiedTime + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorCutoverStatus. +func (in *EtcdMirrorCutoverStatus) DeepCopy() *EtcdMirrorCutoverStatus { + if in == nil { + return nil + } + out := new(EtcdMirrorCutoverStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorDriftInfo) DeepCopyInto(out *EtcdMirrorDriftInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorDriftInfo. +func (in *EtcdMirrorDriftInfo) DeepCopy() *EtcdMirrorDriftInfo { + if in == nil { + return nil + } + out := new(EtcdMirrorDriftInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorEndpoint) DeepCopyInto(out *EtcdMirrorEndpoint) { + *out = *in + if in.EndpointList != nil { + in, out := &in.EndpointList, &out.EndpointList + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ServiceRef != nil { + in, out := &in.ServiceRef, &out.ServiceRef + *out = new(EtcdMirrorServiceRef) + **out = **in + } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(EtcdMirrorTLS) + (*in).DeepCopyInto(*out) + } + if in.Auth != nil { + in, out := &in.Auth, &out.Auth + *out = new(EtcdMirrorAuth) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorEndpoint. +func (in *EtcdMirrorEndpoint) DeepCopy() *EtcdMirrorEndpoint { + if in == nil { + return nil + } + out := new(EtcdMirrorEndpoint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorInitialSyncSpec) DeepCopyInto(out *EtcdMirrorInitialSyncSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorInitialSyncSpec. +func (in *EtcdMirrorInitialSyncSpec) DeepCopy() *EtcdMirrorInitialSyncSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorInitialSyncSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorList) DeepCopyInto(out *EtcdMirrorList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EtcdMirror, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorList. +func (in *EtcdMirrorList) DeepCopy() *EtcdMirrorList { + if in == nil { + return nil + } + out := new(EtcdMirrorList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EtcdMirrorList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorReconciliationSpec) DeepCopyInto(out *EtcdMirrorReconciliationSpec) { + *out = *in + if in.Interval != nil { + in, out := &in.Interval, &out.Interval + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorReconciliationSpec. +func (in *EtcdMirrorReconciliationSpec) DeepCopy() *EtcdMirrorReconciliationSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorReconciliationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorServiceRef) DeepCopyInto(out *EtcdMirrorServiceRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorServiceRef. +func (in *EtcdMirrorServiceRef) DeepCopy() *EtcdMirrorServiceRef { + if in == nil { + return nil + } + out := new(EtcdMirrorServiceRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorSpec) DeepCopyInto(out *EtcdMirrorSpec) { + *out = *in + in.Source.DeepCopyInto(&out.Source) + in.Target.DeepCopyInto(&out.Target) + if in.InitialSync != nil { + in, out := &in.InitialSync, &out.InitialSync + *out = new(EtcdMirrorInitialSyncSpec) + **out = **in + } + in.Sync.DeepCopyInto(&out.Sync) + if in.Checkpoint != nil { + in, out := &in.Checkpoint, &out.Checkpoint + *out = new(EtcdMirrorCheckpointSpec) + **out = **in + } + if in.Reconciliation != nil { + in, out := &in.Reconciliation, &out.Reconciliation + *out = new(EtcdMirrorReconciliationSpec) + (*in).DeepCopyInto(*out) + } + if in.PodTemplate != nil { + in, out := &in.PodTemplate, &out.PodTemplate + *out = new(PodTemplate) + (*in).DeepCopyInto(*out) + } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorSpec. +func (in *EtcdMirrorSpec) DeepCopy() *EtcdMirrorSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorStatus) DeepCopyInto(out *EtcdMirrorStatus) { + *out = *in + if in.InitialSyncStartTime != nil { + in, out := &in.InitialSyncStartTime, &out.InitialSyncStartTime + *out = (*in).DeepCopy() + } + if in.InitialSyncCompletionTime != nil { + in, out := &in.InitialSyncCompletionTime, &out.InitialSyncCompletionTime + *out = (*in).DeepCopy() + } + if in.LastReconciliationTime != nil { + in, out := &in.LastReconciliationTime, &out.LastReconciliationTime + *out = (*in).DeepCopy() + } + if in.LastReconciliationDrift != nil { + in, out := &in.LastReconciliationDrift, &out.LastReconciliationDrift + *out = new(EtcdMirrorDriftInfo) + **out = **in + } + if in.LastStatusSyncTime != nil { + in, out := &in.LastStatusSyncTime, &out.LastStatusSyncTime + *out = (*in).DeepCopy() + } + if in.LastProgressTime != nil { + in, out := &in.LastProgressTime, &out.LastProgressTime + *out = (*in).DeepCopy() + } + if in.Cutover != nil { + in, out := &in.Cutover, &out.Cutover + *out = new(EtcdMirrorCutoverStatus) + (*in).DeepCopyInto(*out) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorStatus. +func (in *EtcdMirrorStatus) DeepCopy() *EtcdMirrorStatus { + if in == nil { + return nil + } + out := new(EtcdMirrorStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorSyncSpec) DeepCopyInto(out *EtcdMirrorSyncSpec) { + *out = *in + if in.ExcludePrefixes != nil { + in, out := &in.ExcludePrefixes, &out.ExcludePrefixes + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.TxnFlushBytes != nil { + in, out := &in.TxnFlushBytes, &out.TxnFlushBytes + x := (*in).DeepCopy() + *out = &x + } + if in.PageBytes != nil { + in, out := &in.PageBytes, &out.PageBytes + x := (*in).DeepCopy() + *out = &x + } + if in.WatchBufferBytes != nil { + in, out := &in.WatchBufferBytes, &out.WatchBufferBytes + x := (*in).DeepCopy() + *out = &x + } + if in.RequestTimeout != nil { + in, out := &in.RequestTimeout, &out.RequestTimeout + *out = new(metav1.Duration) + **out = **in + } + if in.DialTimeout != nil { + in, out := &in.DialTimeout, &out.DialTimeout + *out = new(metav1.Duration) + **out = **in + } + if in.ReconnectBackoff != nil { + in, out := &in.ReconnectBackoff, &out.ReconnectBackoff + *out = new(EtcdMirrorBackoffSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorSyncSpec. +func (in *EtcdMirrorSyncSpec) DeepCopy() *EtcdMirrorSyncSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorSyncSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorTLS) DeepCopyInto(out *EtcdMirrorTLS) { + *out = *in + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(v1.LocalObjectReference) + **out = **in + } + if in.CABundleRef != nil { + in, out := &in.CABundleRef, &out.CABundleRef + *out = new(EtcdMirrorCABundleRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorTLS. +func (in *EtcdMirrorTLS) DeepCopy() *EtcdMirrorTLS { + if in == nil { + return nil + } + out := new(EtcdMirrorTLS) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MemberStatus) DeepCopyInto(out *MemberStatus) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index a04983ec..4f6750c1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -59,6 +59,7 @@ func init() { func main() { var imageRegistry string + var mirrorAgentImage string var metricsAddr string var enableLeaderElection bool var probeAddr string @@ -67,6 +68,9 @@ func main() { var tlsOpts []func(*tls.Config) flag.StringVar(&imageRegistry, "image-registry", "gcr.io/etcd-development/etcd", "The container registry to pull etcd images from. Defaults to gcr.io/etcd-development/etcd.") + flag.StringVar(&mirrorAgentImage, "mirror-agent-image", "", + "Image for EtcdMirror agent Deployments (the operator image itself; the binary ships at /mirror-agent). "+ + "EtcdMirror CRs stay Pending until set.") flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -158,6 +162,14 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "EtcdCluster") os.Exit(1) } + if err = (&controller.EtcdMirrorReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + AgentImage: mirrorAgentImage, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "EtcdMirror") + os.Exit(1) + } // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/cmd/mirror-agent/certexpiry.go b/cmd/mirror-agent/certexpiry.go new file mode 100644 index 00000000..72235e5f --- /dev/null +++ b/cmd/mirror-agent/certexpiry.go @@ -0,0 +1,148 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "crypto/x509" + "encoding/pem" + "fmt" + "os" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// certExpiryRefreshInterval is how often the expiry gauges re-read the cert +// files (mounted Secrets update in place on rotation). +const certExpiryRefreshInterval = 5 * time.Minute + +const ( + certKindClient = "client" + certKindCA = "ca" +) + +// certFile is one PEM file feeding the expiry gauge. +type certFile struct { + side, kind, path string +} + +// certExpiryFiles lists the cert files the flags actually configured: +// kind="client" is the leaf certificate, kind="ca" the effective trust file +// (the bundle wins over the ca file — the same precedence the dial path +// uses). Only configured files produce series. +func certExpiryFiles(sides ...*sideFlags) []certFile { + var out []certFile + for _, s := range sides { + if !s.tls { + // buildTLSInfo rejects material flags without ---tls; this + // guard keeps the gauge from ever vouching for unused files. + continue + } + if s.certFile != "" { + out = append(out, certFile{side: s.side, kind: certKindClient, path: s.certFile}) + } + trust := s.caFile + if s.caBundleFile != "" { + trust = s.caBundleFile + } + if trust != "" { + out = append(out, certFile{side: s.side, kind: certKindCA, path: trust}) + } + } + return out +} + +// certExpiryTracker maintains tls_cert_expiry_timestamp_seconds{side,kind}: +// the EARLIEST NotAfter among each file's certificates (bundle-safe). On a +// read/parse error the series is deleted — a stale expiry gauge must never +// reassure. The expiry lead-window Warning event is the controller's job; +// only the gauge lives in the agent. +type certExpiryTracker struct { + gauge *prometheus.GaugeVec + files []certFile +} + +func newCertExpiryTracker(files []certFile) *certExpiryTracker { + return &certExpiryTracker{ + gauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: metricPrefix + "tls_cert_expiry_timestamp_seconds", + Help: "Earliest NotAfter (unix seconds) among the certificates in the configured " + + "file. Absent while the file is missing or unparseable.", + }, []string{"side", "kind"}), + files: files, + } +} + +// refresh re-reads every configured file and updates the gauges. +func (t *certExpiryTracker) refresh() { + for _, f := range t.files { + expiry, err := earliestNotAfter(f.path) + if err != nil { + setupLog.Error(err, "reading certificate for the expiry gauge", + "side", f.side, "kind", f.kind, "path", f.path) + t.gauge.DeleteLabelValues(f.side, f.kind) + continue + } + t.gauge.WithLabelValues(f.side, f.kind).Set(float64(expiry.Unix())) + } +} + +// loop refreshes on a fixed ticker until ctx is cancelled. +func (t *certExpiryTracker) loop(ctx context.Context) { + tick := time.NewTicker(certExpiryRefreshInterval) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + t.refresh() + } + } +} + +// earliestNotAfter parses every CERTIFICATE PEM block in the file and +// returns the earliest expiry. +func earliestNotAfter(path string) (time.Time, error) { + data, err := os.ReadFile(path) + if err != nil { + return time.Time{}, err + } + var earliest time.Time + for { + var block *pem.Block + block, data = pem.Decode(data) + if block == nil { + break + } + if block.Type != "CERTIFICATE" { + continue + } + cert, cerr := x509.ParseCertificate(block.Bytes) + if cerr != nil { + return time.Time{}, fmt.Errorf("parsing certificate in %s: %w", path, cerr) + } + if earliest.IsZero() || cert.NotAfter.Before(earliest) { + earliest = cert.NotAfter + } + } + if earliest.IsZero() { + return time.Time{}, fmt.Errorf("no CERTIFICATE PEM block in %s", path) + } + return earliest, nil +} diff --git a/cmd/mirror-agent/certexpiry_test.go b/cmd/mirror-agent/certexpiry_test.go new file mode 100644 index 00000000..596f539e --- /dev/null +++ b/cmd/mirror-agent/certexpiry_test.go @@ -0,0 +1,111 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +// writeCertFile writes one self-signed certificate per notAfter to path, as +// a concatenated PEM bundle. +func writeCertFile(t *testing.T, path string, notAfters ...time.Time) { + t.Helper() + pemData := make([]byte, 0, 2048) + for i, notAfter := range notAfters { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(int64(i + 1)), + Subject: pkix.Name{CommonName: "mirror-agent-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: notAfter, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + pemData = append(pemData, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...) + } + require.NoError(t, os.WriteFile(path, pemData, 0o600)) +} + +func TestCertExpiryGauge(t *testing.T) { + dir := t.TempDir() + leafPath := filepath.Join(dir, "tls.crt") + bundlePath := filepath.Join(dir, "ca-bundle.crt") + near := time.Now().Add(24 * time.Hour).Truncate(time.Second) + far := time.Now().Add(90 * 24 * time.Hour).Truncate(time.Second) + writeCertFile(t, leafPath, far) + writeCertFile(t, bundlePath, far, near) // overlapping bundle: two CAs + + tracker := newCertExpiryTracker(certExpiryFiles( + &sideFlags{side: sideSource, tls: true, certFile: leafPath, keyFile: "unused", + caFile: filepath.Join(dir, "ignored-ca.crt"), caBundleFile: bundlePath}, + )) + tracker.refresh() + + // kind="client" reads the leaf; kind="ca" reads the bundle (precedence + // over ca-file) and reports the EARLIEST NotAfter of its certificates. + require.Equal(t, float64(far.Unix()), + testutil.ToFloat64(tracker.gauge.WithLabelValues(sideSource, certKindClient))) + require.Equal(t, float64(near.Unix()), + testutil.ToFloat64(tracker.gauge.WithLabelValues(sideSource, certKindCA))) + + // Rotation: rewrite the leaf, refresh, gauge moves. + rotated := time.Now().Add(48 * time.Hour).Truncate(time.Second) + writeCertFile(t, leafPath, rotated) + tracker.refresh() + require.Equal(t, float64(rotated.Unix()), + testutil.ToFloat64(tracker.gauge.WithLabelValues(sideSource, certKindClient))) + + // A vanished file deletes its series — a stale expiry must not reassure. + require.NoError(t, os.Remove(leafPath)) + tracker.refresh() + require.Equal(t, 1, testutil.CollectAndCount(tracker.gauge), + "only the ca series should remain") +} + +func TestCertExpiryFilesOnlyConfigured(t *testing.T) { + files := certExpiryFiles( + &sideFlags{side: sideSource, tls: true, caFile: "/mnt/ca.crt"}, + &sideFlags{side: sideTarget}, + ) + require.Equal(t, []certFile{{side: sideSource, kind: certKindCA, path: "/mnt/ca.crt"}}, files) + + // A tls=false side never feeds the gauge, even with file flags set + // (buildTLSInfo rejects that combination before the tracker exists). + require.Empty(t, certExpiryFiles( + &sideFlags{side: sideSource, certFile: "/mnt/tls.crt", caFile: "/mnt/ca.crt"})) +} + +func TestEarliestNotAfterNoCertificate(t *testing.T) { + path := filepath.Join(t.TempDir(), "empty.crt") + require.NoError(t, os.WriteFile(path, []byte("not pem"), 0o600)) + _, err := earliestNotAfter(path) + require.ErrorContains(t, err, "no CERTIFICATE PEM block") +} diff --git a/cmd/mirror-agent/config.go b/cmd/mirror-agent/config.go new file mode 100644 index 00000000..49d28c73 --- /dev/null +++ b/cmd/mirror-agent/config.go @@ -0,0 +1,352 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "os" + "strings" + "time" + + "k8s.io/apimachinery/pkg/api/resource" + + "go.etcd.io/etcd-operator/pkg/mirroragent" +) + +const ( + sideSource = "source" + sideTarget = "target" +) + +// stringSliceFlag is a repeatable flag.Value accumulating one entry per +// occurrence. Repeatable rather than comma-separated because etcd key +// prefixes may legally contain commas. +type stringSliceFlag []string + +func (s *stringSliceFlag) String() string { return strings.Join(*s, ",") } + +func (s *stringSliceFlag) Set(v string) error { + *s = append(*s, v) + return nil +} + +// sideFlags is the per-side (source/target) flag surface: endpoints, TLS +// material paths, and auth credential paths. Secrets arrive only as +// mounted-file paths — never as flag values or environment variables. +type sideFlags struct { + side string + + endpoints string + + // tls mirrors has(spec..tls) on the EtcdMirror CR: true dials TLS + // even with no file flags (server-auth against the system trust roots). + tls bool + certFile string + keyFile string + caFile string + caBundleFile string + serverName string + insecureSkipVerify bool + + usernameFile string + passwordFile string +} + +// agentFlags is the binary's whole flag surface. Each field maps onto the +// EtcdMirror CRD field of the same name; semantics are pinned there. +type agentFlags struct { + linkUID string + epoch int64 + mode string + + sourcePrefix string + targetPrefix string + destPrefix string + excludePrefixes stringSliceFlag + + initialSyncMode string + startRevision int64 + checkpointKey string + + maxTxnOps int + txnFlushBytes string + pageBytes string + watchBufferBytes string + pageKeyLimit int + maxOpsPerSecond int + requestTimeout time.Duration + dialTimeout time.Duration + + backoffInitialDelay time.Duration + backoffMaxDelay time.Duration + + reconcileEnabled bool + reconcileInterval time.Duration + reconcileDeleteOrphans bool + + httpBindAddress string + + source sideFlags + target sideFlags +} + +// newAgentFlags registers every mirror-agent flag on fs and returns the +// struct they populate. +func newAgentFlags(fs *flag.FlagSet) *agentFlags { + f := &agentFlags{} + fs.StringVar(&f.linkUID, "link-uid", "", + "Unique identifier of this mirror link, stamped into the fence key "+ + "(typically the EtcdMirror object's UID). Required.") + fs.Int64Var(&f.epoch, "epoch", 0, + "This agent generation within the link, bumped by the supervisor on each re-deploy. Must be >= 1. Required.") + fs.StringVar(&f.mode, "mode", string(mirroragent.ModeSync), + "Operating mode: Sync (continuous replication) or Drain (cutover drain).") + fs.StringVar(&f.sourcePrefix, "source-prefix", "", + "Source key prefix to mirror; empty means the whole keyspace.") + fs.StringVar(&f.targetPrefix, "target-prefix", "", + "Target prefix under which mirrored keys land.") + fs.StringVar(&f.destPrefix, "dest-prefix", "", + "Middle term of the rewrite formula key' = target-prefix + dest-prefix + TrimPrefix(key, source-prefix).") + fs.Var(&f.excludePrefixes, "exclude-prefix", + "Source key prefix skipped entirely (not scanned, watched, counted, or pruned). Repeatable.") + fs.StringVar(&f.initialSyncMode, "initial-sync-mode", "", + "Pre-existing destination-key policy at genesis: RequireEmpty (default), Overwrite, or OverwriteAndPrune.") + fs.Int64Var(&f.startRevision, "start-revision", 0, + "When > 0, skip the genesis scan and tail from this source revision + 1 "+ + "(requires initial-sync-mode Overwrite or OverwriteAndPrune).") + fs.StringVar(&f.checkpointKey, "checkpoint-key", "", + "Reserved checkpoint/fence key on the target; empty selects the engine default "+ + "(effective destination prefix + a \\x00-prefixed suffix). Only pass when spec.checkpoint.key is set.") + fs.IntVar(&f.maxTxnOps, "max-txn-ops", 0, + "Max operations per target Txn including the reserved checkpoint slot; 0 selects the engine default (128).") + fs.StringVar(&f.txnFlushBytes, "txn-flush-bytes", "", + "Byte watermark at which a batch is flushed, as a Kubernetes quantity (e.g. 1Mi); empty selects the default.") + fs.StringVar(&f.pageBytes, "page-bytes", "", + "Byte bound per source scan page, as a Kubernetes quantity; empty selects the default.") + fs.StringVar(&f.watchBufferBytes, "watch-buffer-bytes", "", + "Replay-buffer byte bound while the genesis scan runs, as a Kubernetes quantity; empty selects the default.") + fs.IntVar(&f.pageKeyLimit, "page-key-limit", 0, + "Key bound per source scan page; 0 selects the engine default (512).") + fs.IntVar(&f.maxOpsPerSecond, "max-ops-per-second", 0, + "Target write rate limit (puts+deletes/sec); 0 means unlimited.") + fs.DurationVar(&f.requestTimeout, "request-timeout", 0, + "Per-RPC deadline for every unary call on both sides; 0 selects the engine default (30s).") + fs.DurationVar(&f.dialTimeout, "dial-timeout", 10*time.Second, + "Bound on establishing the initial client connection to each side.") + fs.DurationVar(&f.backoffInitialDelay, "backoff-initial-delay", 0, + "Initial delay of the connection-error retry curve; 0 selects the engine default (1s).") + fs.DurationVar(&f.backoffMaxDelay, "backoff-max-delay", 0, + "Max delay of the connection-error retry curve; 0 selects the engine default (30s).") + fs.BoolVar(&f.reconcileEnabled, "reconcile-enabled", false, + "Enable the periodic full diff-and-repair pass.") + fs.DurationVar(&f.reconcileInterval, "reconcile-interval", 0, + "Interval between periodic reconciliation passes; 0 with --reconcile-enabled selects the default (1h).") + fs.BoolVar(&f.reconcileDeleteOrphans, "reconcile-delete-orphans", false, + "Allow the periodic pass to delete target keys with no source counterpart.") + fs.StringVar(&f.httpBindAddress, "http-bind-address", ":8080", + "Listen address for /statusz, /healthz, /readyz and /metrics.") + registerSideFlags(fs, sideSource, &f.source) + registerSideFlags(fs, sideTarget, &f.target) + return f +} + +// registerSideFlags registers one side's flag block; side is "source" or +// "target", matching spec.source / spec.target on the EtcdMirror CR. +func registerSideFlags(fs *flag.FlagSet, side string, out *sideFlags) { + out.side = side + p := func(name string) string { return side + "-" + name } + fs.StringVar(&out.endpoints, p("endpoints"), "", + "Comma-separated etcd client endpoints of the "+side+" cluster. Schemeless endpoints get the "+ + "scheme implied by --"+p("tls")+"; a scheme contradicting that flag is an error. Required.") + fs.BoolVar(&out.tls, p("tls"), false, + "Dial the "+side+" side with TLS; true with no file flags means server-auth TLS against the "+ + "system trust roots (mirrors an empty spec tls block).") + fs.StringVar(&out.certFile, p("cert-file"), "", + "Path to the "+side+" client certificate for mTLS. Requires --"+p("key-file")+".") + fs.StringVar(&out.keyFile, p("key-file"), "", + "Path to the "+side+" client key for mTLS. Requires --"+p("cert-file")+".") + fs.StringVar(&out.caFile, p("ca-file"), "", + "Path to the PEM trust anchors verifying "+side+" servers (the mounted secret's ca.crt).") + fs.StringVar(&out.caBundleFile, p("ca-bundle-file"), "", + "Path to a separate PEM CA bundle; takes precedence over --"+p("ca-file")+" (mirrors tls.caBundleRef).") + fs.StringVar(&out.serverName, p("server-name"), "", + "TLS ServerName (SNI) override used to verify every "+side+" endpoint.") + fs.BoolVar(&out.insecureSkipVerify, p("insecure-skip-verify"), false, + "Disable "+side+" server certificate verification. The supervisor only sets this when the "+ + "EtcdMirror acknowledged the risk (insecureSkipVerifyAcknowledgeRisk).") + fs.StringVar(&out.usernameFile, p("username-file"), "", + "Path to a file holding the "+side+" etcd RBAC username, read once at startup. "+ + "Requires --"+p("password-file")+".") + fs.StringVar(&out.passwordFile, p("password-file"), "", + "Path to a file holding the "+side+" etcd RBAC password, read once at startup. "+ + "Requires --"+p("username-file")+".") +} + +// buildConfig translates flags into the engine Config. Only translation-level +// checks live here (quantity parsing, required flags); everything else is +// left to mirroragent.New → Config.Validate — no duplicated validation. In +// particular a negative --watch-buffer-bytes passes through and is rejected +// by Validate (lockstep: 0 = engine default, negative rejected). +func buildConfig(f *agentFlags) (mirroragent.Config, error) { + if f.linkUID == "" { + return mirroragent.Config{}, fmt.Errorf("--link-uid is required") + } + if f.epoch < 1 { + return mirroragent.Config{}, fmt.Errorf("--epoch must be >= 1, got %d", f.epoch) + } + txnFlushBytes, err := parseQuantityBytes("txn-flush-bytes", f.txnFlushBytes) + if err != nil { + return mirroragent.Config{}, err + } + pageBytes, err := parseQuantityBytes("page-bytes", f.pageBytes) + if err != nil { + return mirroragent.Config{}, err + } + watchBufferBytes, err := parseQuantityBytes("watch-buffer-bytes", f.watchBufferBytes) + if err != nil { + return mirroragent.Config{}, err + } + return mirroragent.Config{ + LinkUID: f.linkUID, + Epoch: f.epoch, + Mode: mirroragent.Mode(f.mode), + SourcePrefix: f.sourcePrefix, + TargetPrefix: f.targetPrefix, + DestPrefix: f.destPrefix, + ExcludePrefixes: f.excludePrefixes, + InitialSyncMode: mirroragent.InitialSyncMode(f.initialSyncMode), + StartRevision: f.startRevision, + CheckpointKey: f.checkpointKey, + MaxTxnOps: f.maxTxnOps, + TxnFlushBytes: txnFlushBytes, + PageKeyLimit: f.pageKeyLimit, + PageBytes: pageBytes, + MaxOpsPerSecond: f.maxOpsPerSecond, + RequestTimeout: f.requestTimeout, + BackoffInitialDelay: f.backoffInitialDelay, + BackoffMaxDelay: f.backoffMaxDelay, + ReconcileInterval: reconcilePeriod(f.reconcileEnabled, f.reconcileInterval), + ReconcileDeleteOrphans: f.reconcileDeleteOrphans, + WatchBufferBytes: watchBufferBytes, + }, nil +} + +// parseQuantityBytes parses a Kubernetes resource.Quantity string ("16Mi", +// "1048576") into bytes. Empty means unset (0 → engine default). +func parseQuantityBytes(name, s string) (int64, error) { + if s == "" { + return 0, nil + } + q, err := resource.ParseQuantity(s) + if err != nil { + return 0, fmt.Errorf("--%s: invalid quantity %q: %w", name, s, err) + } + return q.Value(), nil +} + +// reconcilePeriod is the spec.reconciliation → Config.ReconcileInterval +// translation (see DefaultReconcilePeriod in pkg/mirroragent): disabled → 0 +// regardless of interval; enabled with no interval → the 1h default. +func reconcilePeriod(enabled bool, interval time.Duration) time.Duration { + switch { + case !enabled: + return 0 + case interval > 0: + return interval + default: + return mirroragent.DefaultReconcilePeriod + } +} + +// normalizeEndpoints splits the comma-separated list (URLs cannot contain +// commas) and makes the CRD's scheme/TLS agreement true by construction: +// schemeless endpoints get the scheme implied by the tls flag, and a scheme +// contradicting the flag is an error — the binary mirror of the CEL rules on +// EtcdMirrorEndpoint. +func normalizeEndpoints(side, list string, useTLS bool) ([]string, error) { + var out []string + for _, ep := range strings.Split(list, ",") { + ep = strings.TrimSpace(ep) + if ep == "" { + continue + } + switch { + case strings.HasPrefix(ep, "http://"): + if useTLS { + return nil, fmt.Errorf( + "--%s-endpoints: http:// endpoint %q conflicts with --%s-tls", side, ep, side) + } + case strings.HasPrefix(ep, "https://"): + if !useTLS { + return nil, fmt.Errorf( + "--%s-endpoints: https:// endpoint %q requires --%s-tls", side, ep, side) + } + case useTLS: + ep = "https://" + ep + default: + ep = "http://" + ep + } + out = append(out, ep) + } + if len(out) == 0 { + return nil, fmt.Errorf("--%s-endpoints is required", side) + } + return out, nil +} + +// readAuthFiles reads the etcd RBAC username/password from mounted files, +// once at startup: clientv3 fixes credentials at client construction and +// transparently re-authenticates on token expiry, so there is nothing to +// re-read. One trailing newline is stripped. PRECEDENCE (per the +// EtcdMirrorAuth contract): when both a client certificate and auth are +// supplied, etcd uses the token identity, not the certificate CN — the auth +// user must hold the range-scoped role. +func readAuthFiles(side, usernameFile, passwordFile string) (string, string, error) { + if (usernameFile == "") != (passwordFile == "") { + return "", "", fmt.Errorf( + "--%s-username-file and --%s-password-file must be set together", side, side) + } + if usernameFile == "" { + return "", "", nil + } + username, err := readSecretFile(usernameFile) + if err != nil { + return "", "", fmt.Errorf("--%s-username-file: %w", side, err) + } + password, err := readSecretFile(passwordFile) + if err != nil { + return "", "", fmt.Errorf("--%s-password-file: %w", side, err) + } + // clientv3 silently skips authentication unless BOTH are non-empty, so an + // accidentally empty mounted file must fail here, not dial unauthenticated. + if username == "" { + return "", "", fmt.Errorf("--%s-username-file %s is empty", side, usernameFile) + } + if password == "" { + return "", "", fmt.Errorf("--%s-password-file %s is empty", side, passwordFile) + } + return username, password, nil +} + +func readSecretFile(path string) (string, error) { + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + return strings.TrimSuffix(strings.TrimSuffix(string(b), "\n"), "\r"), nil +} diff --git a/cmd/mirror-agent/config_test.go b/cmd/mirror-agent/config_test.go new file mode 100644 index 00000000..9be0e140 --- /dev/null +++ b/cmd/mirror-agent/config_test.go @@ -0,0 +1,184 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd-operator/pkg/mirroragent" +) + +// parseArgs runs the real flag registration over args. +func parseArgs(t *testing.T, args ...string) *agentFlags { + t.Helper() + fs := flag.NewFlagSet("mirror-agent-test", flag.ContinueOnError) + fs.SetOutput(io.Discard) + f := newAgentFlags(fs) + require.NoError(t, fs.Parse(args)) + return f +} + +func minimalArgs(extra ...string) []string { + return append([]string{ + "--link-uid=test-link", "--epoch=1", + "--source-endpoints=127.0.0.1:2379", "--target-endpoints=127.0.0.1:2380", + }, extra...) +} + +func TestBuildConfigDefaults(t *testing.T) { + f := parseArgs(t, minimalArgs()...) + cfg, err := buildConfig(f) + require.NoError(t, err) + + require.Equal(t, "test-link", cfg.LinkUID) + require.Equal(t, int64(1), cfg.Epoch) + require.Equal(t, mirroragent.ModeSync, cfg.Mode) + // Only what was given is carried; zeros select engine defaults. + require.Empty(t, cfg.InitialSyncMode) + require.Empty(t, cfg.CheckpointKey) + require.Zero(t, cfg.MaxTxnOps) + require.Zero(t, cfg.TxnFlushBytes) + require.Zero(t, cfg.PageBytes) + require.Zero(t, cfg.PageKeyLimit) + require.Zero(t, cfg.WatchBufferBytes) + require.Zero(t, cfg.RequestTimeout) + require.Zero(t, cfg.ReconcileInterval) + require.Equal(t, 10*time.Second, f.dialTimeout) + + // Engine defaulting+validation accepts the translated config. + _, err = mirroragent.New(cfg, nil, nil) + require.NoError(t, err) +} + +func TestBuildConfigRequiredFlags(t *testing.T) { + _, err := buildConfig(parseArgs(t, "--epoch=1")) + require.ErrorContains(t, err, "--link-uid") + _, err = buildConfig(parseArgs(t, "--link-uid=x")) + require.ErrorContains(t, err, "--epoch") +} + +func TestBuildConfigQuantities(t *testing.T) { + f := parseArgs(t, minimalArgs( + "--txn-flush-bytes=16Mi", "--page-bytes=1048576", "--watch-buffer-bytes=32Mi")...) + cfg, err := buildConfig(f) + require.NoError(t, err) + require.Equal(t, int64(16777216), cfg.TxnFlushBytes) + require.Equal(t, int64(1048576), cfg.PageBytes) + require.Equal(t, int64(33554432), cfg.WatchBufferBytes) + + _, err = buildConfig(parseArgs(t, minimalArgs("--page-bytes=garbage")...)) + require.ErrorContains(t, err, "--page-bytes") + + // Lockstep with the engine: a negative watch buffer passes translation + // (0 = default there, not here) and is rejected by Config.Validate. + cfg, err = buildConfig(parseArgs(t, minimalArgs("--watch-buffer-bytes=-1")...)) + require.NoError(t, err) + require.Equal(t, int64(-1), cfg.WatchBufferBytes) + _, err = mirroragent.New(cfg, nil, nil) + require.ErrorContains(t, err, "watchBufferBytes") +} + +func TestBuildConfigReconcileTranslation(t *testing.T) { + cases := []struct { + name string + enabled bool + interv time.Duration + want time.Duration + }{ + {"enabled default", true, 0, mirroragent.DefaultReconcilePeriod}, + {"enabled explicit", true, 45 * time.Minute, 45 * time.Minute}, + {"disabled ignores interval", false, 45 * time.Minute, 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, reconcilePeriod(tc.enabled, tc.interv)) + }) + } +} + +func TestBuildConfigExcludePrefixRepeatable(t *testing.T) { + f := parseArgs(t, minimalArgs( + "--exclude-prefix=/registry/events/", "--exclude-prefix=/a,b/")...) + cfg, err := buildConfig(f) + require.NoError(t, err) + require.Equal(t, []string{"/registry/events/", "/a,b/"}, cfg.ExcludePrefixes) +} + +func TestNormalizeEndpoints(t *testing.T) { + got, err := normalizeEndpoints(sideSource, "a:2379, b:2379", true) + require.NoError(t, err) + require.Equal(t, []string{"https://a:2379", "https://b:2379"}, got) + + got, err = normalizeEndpoints(sideSource, "a:2379", false) + require.NoError(t, err) + require.Equal(t, []string{"http://a:2379"}, got) + + got, err = normalizeEndpoints(sideTarget, "https://a:2379,http://b:2379", false) + require.Nil(t, got) + require.ErrorContains(t, err, "requires --target-tls") + + got, err = normalizeEndpoints(sideTarget, "http://a:2379", true) + require.Nil(t, got) + require.ErrorContains(t, err, "conflicts with --target-tls") + + got, err = normalizeEndpoints(sideSource, "https://a:2379,https://b:2379", true) + require.NoError(t, err) + require.Equal(t, []string{"https://a:2379", "https://b:2379"}, got) + + _, err = normalizeEndpoints(sideSource, "", false) + require.ErrorContains(t, err, "--source-endpoints is required") +} + +func TestReadAuthFiles(t *testing.T) { + dir := t.TempDir() + userFile := filepath.Join(dir, "username") + passFile := filepath.Join(dir, "password") + require.NoError(t, os.WriteFile(userFile, []byte("mirror-user\n"), 0o600)) + require.NoError(t, os.WriteFile(passFile, []byte("s3cret"), 0o600)) + + user, pass, err := readAuthFiles(sideSource, userFile, passFile) + require.NoError(t, err) + require.Equal(t, "mirror-user", user, "one trailing newline is stripped") + require.Equal(t, "s3cret", pass) + + user, pass, err = readAuthFiles(sideSource, "", "") + require.NoError(t, err) + require.Empty(t, user) + require.Empty(t, pass) + + _, _, err = readAuthFiles(sideTarget, userFile, "") + require.ErrorContains(t, err, "must be set together") + + _, _, err = readAuthFiles(sideSource, filepath.Join(dir, "missing"), passFile) + require.Error(t, err) + + // Empty content (or a lone newline) would make clientv3 silently skip + // authentication; it must fail startup instead. + emptyFile := filepath.Join(dir, "empty") + require.NoError(t, os.WriteFile(emptyFile, []byte("\n"), 0o600)) + _, _, err = readAuthFiles(sideSource, emptyFile, passFile) + require.ErrorContains(t, err, "--source-username-file "+emptyFile+" is empty") + _, _, err = readAuthFiles(sideTarget, userFile, emptyFile) + require.ErrorContains(t, err, "--target-password-file "+emptyFile+" is empty") +} diff --git a/cmd/mirror-agent/live_test.go b/cmd/mirror-agent/live_test.go new file mode 100644 index 00000000..c8452396 --- /dev/null +++ b/cmd/mirror-agent/live_test.go @@ -0,0 +1,159 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + crlog "sigs.k8s.io/controller-runtime/pkg/log" + + "go.etcd.io/etcd-operator/pkg/mirroragent" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" +) + +// startTestEtcd boots an embedded etcd on loopback and returns its client +// URL (http://...). +func startTestEtcd(t *testing.T) string { + t.Helper() + loopbackURL := func() *url.URL { + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer func() { require.NoError(t, l.Close()) }() + u, err := url.Parse("http://" + l.Addr().String()) + require.NoError(t, err) + return u + } + cfg := embed.NewConfig() + cfg.Dir = t.TempDir() + cfg.Logger, cfg.LogLevel = "zap", "error" + client, peer := loopbackURL(), loopbackURL() + cfg.ListenClientUrls, cfg.AdvertiseClientUrls = []url.URL{*client}, []url.URL{*client} + cfg.ListenPeerUrls, cfg.AdvertisePeerUrls = []url.URL{*peer}, []url.URL{*peer} + cfg.InitialCluster = cfg.Name + "=" + peer.String() + etcd, err := embed.StartEtcd(cfg) + require.NoError(t, err) + t.Cleanup(etcd.Close) + select { + case <-etcd.Server.ReadyNotify(): + case <-time.After(time.Minute): + t.Fatal("embedded etcd took too long to start") + } + return client.String() +} + +// TestLiveAgentEndpoints exercises the real buildConfig → clients → New → +// Run → HTTP path end to end against two embedded etcds, cleartext. +func TestLiveAgentEndpoints(t *testing.T) { + crlog.SetLogger(logr.Discard()) + srcURL := startTestEtcd(t) + dstURL := startTestEtcd(t) + + src, err := clientv3.New(mirroragent.NewClientConfig([]string{srcURL}, nil, 5*time.Second)) + require.NoError(t, err) + t.Cleanup(func() { _ = src.Close() }) + for i := range 5 { + _, err = src.Put(t.Context(), fmt.Sprintf("/live/key-%d", i), fmt.Sprintf("val-%d", i)) + require.NoError(t, err) + } + + f := parseArgs(t, + "--link-uid=live-test", "--epoch=1", + "--source-endpoints="+srcURL, "--target-endpoints="+dstURL, + "--source-prefix=/live/", "--target-prefix=/mirror/", + "--request-timeout=5s", "--dial-timeout=5s", + "--http-bind-address=127.0.0.1:0", + ) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + addrCh := make(chan net.Addr, 1) + runDone := make(chan error, 1) + go func() { runDone <- run(ctx, f, func(a net.Addr) { addrCh <- a }) }() + + var base string + select { + case addr := <-addrCh: + base = "http://" + addr.String() + case <-time.After(15 * time.Second): + t.Fatal("run never bound the HTTP listener") + } + + // Poll /statusz until the engine reaches steady state with applies. + // assert (not require) so the failure message can read snap AFTER the + // polling — require.Eventually's msgAndArgs are copied before it. + var snap mirroragent.Snapshot + if !assert.Eventually(t, func() bool { + snap = getStatusz(t, base) + return snap.Phase == mirroragent.PhaseSyncing && snap.Watermark > 0 && snap.KeysAppliedTotal > 0 + }, 30*time.Second, 100*time.Millisecond) { + t.Fatalf("engine never reached steady state; last snapshot: %+v", snap) + } + require.NotZero(t, snap.SourceClusterID) + require.NotZero(t, snap.TargetClusterID) + require.False(t, snap.LastProgressTime.IsZero()) + + resp, err := http.Get(base + "/readyz") + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusOK, resp.StatusCode) + + resp, err = http.Get(base + "/metrics") + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + metrics := string(body) + require.Contains(t, metrics, "etcd_mirror_agent_watermark_revision") + require.Contains(t, metrics, "etcd_mirror_agent_keys_applied_total") + require.NotContains(t, metrics, "etcd_mirror_agent_keys_applied_total 0\n", + "keys_applied_total must be nonzero after the scan") + + // SIGTERM-equivalent: cancel the run context; the engine stops and the + // server drains, and the whole run resolves to the exit-0 path (nil). + cancel() + select { + case err := <-runDone: + require.NoError(t, err) + case <-time.After(20 * time.Second): + t.Fatal("run did not return after cancel") + } +} + +func getStatusz(t *testing.T, base string) mirroragent.Snapshot { + t.Helper() + resp, err := http.Get(base + "/statusz") + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + require.Equal(t, http.StatusOK, resp.StatusCode) + require.True(t, strings.HasPrefix(resp.Header.Get("Content-Type"), "application/json")) + var snap mirroragent.Snapshot + require.NoError(t, json.NewDecoder(resp.Body).Decode(&snap)) + return snap +} diff --git a/cmd/mirror-agent/main.go b/cmd/mirror-agent/main.go new file mode 100644 index 00000000..5d6713fa --- /dev/null +++ b/cmd/mirror-agent/main.go @@ -0,0 +1,238 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Command mirror-agent runs one pkg/mirroragent replication engine — one +// EtcdMirror link, one process. The operator's EtcdMirror controller renders +// a Deployment running this binary; it can equally be run by hand against +// two reachable etcd clusters. The binary talks only to the two etcd +// clusters: it has zero Kubernetes API dependency. +// +// # Configuration +// +// Everything arrives as flags (the repo's operator convention); secrets — +// TLS material and RBAC credentials — arrive only as mounted-file PATHS +// (---cert-file, ---username-file, ...), never as flag values or +// environment variables. Flag semantics mirror the EtcdMirror CRD fields of +// the same names; quantities (--txn-flush-bytes, --page-bytes, +// --watch-buffer-bytes) accept Kubernetes resource.Quantity strings. +// Unset/zero flags select the engine defaults; the binary validates only +// what the engine cannot (required flags, quantity syntax, scheme/TLS +// agreement, file pairing) and leaves the rest to Config.Validate. +// +// TLS is built from file paths via transport.TLSInfo, so the client leaf +// pair is re-read from the mount on every handshake: certificate rotation +// needs no restart. ---ca-bundle-file takes precedence over +// ---ca-file, mirroring tls.caBundleRef. ---insecure-skip-verify +// has no acknowledge-risk companion flag — that ceremony is CRD/manifest +// UX; the supervisor only passes the flag when the CR acknowledged the +// risk, and the binary logs a standing warning. Auth username/password are +// read once at startup (clientv3 re-auths transparently); the token +// identity wins over the certificate CN when both are supplied. +// +// # Endpoints +// +// One listener (--http-bind-address) serves: +// +// - /statusz — JSON dump of the engine Snapshot (the controller's poll +// surface). +// - /healthz — process liveness, always 200. +// - /readyz — 200 once the engine is past Connecting and not Failed; +// Degraded (transient backoff) and Drained (terminal success) are ready. +// - /metrics — Prometheus (etcd_mirror_agent_* plus process/Go runtime). +// +// # Lifecycle and exit codes +// +// When the engine's Run returns — drain complete or permanent failure — the +// process does NOT exit: it lingers, serving /statusz, /metrics and /readyz, +// so the controller can read the terminal state (Drained + the cutover +// block, or Failed + lastError). Crash-looping would hide that state and +// re-run genesis under the same epoch; restart/epoch-bump policy belongs to +// the supervisor. On SIGINT/SIGTERM the engine context is cancelled, Run is +// awaited, and the HTTP server drains (10s bound). +// +// Exit codes: 0 = clean signal shutdown (engine cancelled mid-run, or it had +// drained); 1 = startup failure or the engine ended in permanent failure by +// the time the signal arrived; 2 = flag-parse errors (stdlib flag). Any +// non-cancellation error escaping Run is permanent by construction — +// transient, throttle and quota conditions are retried inside the engine and +// never escape. +package main + +import ( + "context" + "crypto/tls" + "errors" + "flag" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + crlog "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "go.etcd.io/etcd-operator/pkg/mirroragent" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// httpShutdownTimeout bounds the HTTP server drain after the engine stopped. +const httpShutdownTimeout = 10 * time.Second + +var setupLog = crlog.Log.WithName("mirror-agent") + +func main() { + f := newAgentFlags(flag.CommandLine) + opts := zap.Options{Development: false} + opts.BindFlags(flag.CommandLine) + flag.Parse() + crlog.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + // Once the first signal cancels ctx, restore the default disposition so a + // second SIGINT/SIGTERM kills a wedged shutdown — the same escape hatch + // ctrl.SetupSignalHandler gives the manager binary. + context.AfterFunc(ctx, stop) + if err := run(ctx, f, nil); err != nil { + setupLog.Error(err, "mirror-agent failed") + os.Exit(1) + } +} + +// run wires flags → clients → engine → HTTP and blocks until ctx is +// cancelled. onListen (optional) observes the bound HTTP address, for tests +// binding port 0. A non-nil return is a startup failure or a permanent +// engine failure; cancellation mid-run and a completed drain return nil. +func run(ctx context.Context, f *agentFlags, onListen func(net.Addr)) error { + cfg, err := buildConfig(f) + if err != nil { + return err + } + src, err := buildClient(&f.source, f.dialTimeout) + if err != nil { + return err + } + defer func() { _ = src.Close() }() + dst, err := buildClient(&f.target, f.dialTimeout) + if err != nil { + return err + } + defer func() { _ = dst.Close() }() + + agent, err := mirroragent.New(cfg, src, dst) + if err != nil { + return err + } + + tracker := newCertExpiryTracker(certExpiryFiles(&f.source, &f.target)) + tracker.refresh() + go tracker.loop(ctx) + + listener, err := net.Listen("tcp", f.httpBindAddress) + if err != nil { + return fmt.Errorf("listening on %s: %w", f.httpBindAddress, err) + } + if onListen != nil { + onListen(listener.Addr()) + } + srv := &http.Server{ + Handler: newMux(agent.Snapshot, newRegistry(agent.Snapshot, tracker.gauge)), + ReadHeaderTimeout: readHeaderTimeout, + } + go func() { + if serr := srv.Serve(listener); serr != nil && !errors.Is(serr, http.ErrServerClosed) { + setupLog.Error(serr, "http server failed") + } + }() + + setupLog.Info("starting replication engine", "linkUID", cfg.LinkUID, "epoch", cfg.Epoch, + "mode", cfg.Mode, "httpBindAddress", listener.Addr().String()) + runDone := make(chan error, 1) + go func() { runDone <- agent.Run(ctx) }() + + var runErr error + select { + case runErr = <-runDone: + // Guard against the race where the signal and Run's return are + // simultaneous and select picked this branch: a cancelled run is a + // clean shutdown, not a terminal state to linger on and log about. + if ctx.Err() == nil && !isCancellation(runErr) { + // Terminal engine state (drained or permanent failure): LINGER, + // keep serving the observability surface until the supervisor + // signals — see the package doc's lifecycle contract. + if runErr == nil { + setupLog.Info("drain completed; serving terminal status until signalled") + } else { + setupLog.Error(runErr, "engine failed permanently; serving terminal status until signalled") + } + <-ctx.Done() + } + case <-ctx.Done(): + runErr = <-runDone + } + + shutCtx, cancel := context.WithTimeout(context.Background(), httpShutdownTimeout) + defer cancel() + _ = srv.Shutdown(shutCtx) + + if runErr != nil && !isCancellation(runErr) { + return fmt.Errorf("engine failed permanently: %w", runErr) + } + return nil +} + +// isCancellation reports whether Run's error is context cancellation — a +// clean signal shutdown, never a permanent engine failure. +func isCancellation(err error) bool { + return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) +} + +// buildClient dials one side per its flags: normalized endpoints, TLS from +// mounted file paths (see buildTLSInfo), auth from mounted files, and the +// keepalive settings the engine mandates (NewClientConfig). +func buildClient(side *sideFlags, dialTimeout time.Duration) (*clientv3.Client, error) { + endpoints, err := normalizeEndpoints(side.side, side.endpoints, side.tls) + if err != nil { + return nil, err + } + info, tlsEnabled, err := buildTLSInfo(side) + if err != nil { + return nil, err + } + var tlsCfg *tls.Config + if tlsEnabled { + if side.insecureSkipVerify { + setupLog.Info("WARNING: TLS server certificate verification is DISABLED", "side", side.side) + } + tlsCfg, err = info.ClientConfig() + if err != nil { + return nil, fmt.Errorf("building %s TLS config: %w", side.side, err) + } + } + cc := mirroragent.NewClientConfig(endpoints, tlsCfg, dialTimeout) + cc.Username, cc.Password, err = readAuthFiles(side.side, side.usernameFile, side.passwordFile) + if err != nil { + return nil, err + } + cli, err := clientv3.New(cc) + if err != nil { + return nil, fmt.Errorf("creating %s client: %w", side.side, err) + } + return cli, nil +} diff --git a/cmd/mirror-agent/metrics.go b/cmd/mirror-agent/metrics.go new file mode 100644 index 00000000..942a95e1 --- /dev/null +++ b/cmd/mirror-agent/metrics.go @@ -0,0 +1,204 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "slices" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + + "go.etcd.io/etcd-operator/pkg/mirroragent" +) + +// metricPrefix is the agent metric namespace (the operator's controller +// metrics live elsewhere; the agent is not a controller-runtime manager). +const metricPrefix = "etcd_mirror_agent_" + +var ( + allPhases = []mirroragent.Phase{ + mirroragent.PhaseConnecting, mirroragent.PhaseInitialSync, + mirroragent.PhaseSyncing, mirroragent.PhaseDegraded, + mirroragent.PhaseFailed, mirroragent.PhaseDrained, + } + allClasses = []mirroragent.Class{ + mirroragent.ClassTransient, mirroragent.ClassThrottle, + mirroragent.ClassResync, mirroragent.ClassQuota, + mirroragent.ClassPermanent, + } + // allResyncReasons pre-initializes every forced_resync_total series at 0: + // forced resyncs are rare singular events, and rate()/increase() cannot + // see a counter's first sample, so a series born at 1 hides the resync + // that usually matters most. + allResyncReasons = []mirroragent.ResyncReason{ + mirroragent.ResyncReasonCompacted, mirroragent.ResyncReasonClusterIDMismatch, + } +) + +func newDesc(name, help string, labels ...string) *prometheus.Desc { + return prometheus.NewDesc(metricPrefix+name, help, labels, nil) +} + +var ( + descPhase = newDesc("phase", + "One-hot agent phase (1 on the current phase).", "phase") + descWatermark = newDesc("watermark_revision", + "Checkpoint watermark: the source revision through which the target is caught up.") + descSourceRevision = newDesc("source_revision", + "Source cluster revision as of the last watch header.") + descLag = newDesc("lag_revisions", + "max(0, source_revision - watermark). Overstates lag for prefix-scoped mirrors "+ + "(the source revision is cluster-global). Absent until the source revision is known.") + descSinceProgress = newDesc("seconds_since_last_progress", + "Seconds since the watermark last advanced. Absent until first progress.") + descKeysApplied = newDesc("keys_applied_total", + "Data operations (puts and deletes) committed to the target in fenced Txns.") + descInitialSyncKeys = newDesc("initial_sync_keys", + "Keys copied so far by the genesis scan.") + descInitialSyncExpected = newDesc("initial_sync_expected_keys", + "Expected key total for the genesis scan (the progress denominator).") + descLeaseBackedKeys = newDesc("lease_backed_keys", + "Mirrored keys whose source copy is lease-backed.") + descForcedResync = newDesc("forced_resync_total", + "Forced resyncs by trigger reason.", "reason") + descScanRestart = newDesc("scan_restart_total", + "Genesis scan attempts aborted and restarted from a fresh R0 "+ + "(cause in /statusz lastScanRestartCause).") + descResyncLoop = newDesc("resync_loop_detected", + "1 while the resync-loop livelock detector is latched.") + descThrottled = newDesc("throttled", + "1 while the target is rejecting the write rate (backoff engaged).") + descQuotaExhausted = newDesc("quota_exhausted", + "1 while the target reports NOSPACE (parked on the quota probe).") + descCompacted = newDesc("compacted", + "1 while recovering from source compaction outrunning the watch.") + descCutoverReady = newDesc("cutover_ready", + "1 once the drain completed and the fence role is Primary.") + descLastErrorClass = newDesc("last_error_class", + "One-hot class of the most recent failure (all 0 when the last attempt succeeded).", + "class") + descSourceKeys = newDesc("source_keys", + "In-scope source key count from the most recent reconciliation/verification pass.") + descTargetKeys = newDesc("target_keys", + "In-scope target key count from the most recent reconciliation/verification pass.") + descDriftMissing = newDesc("drift_missing_keys", + "Keys missing on the target per the last full reconciliation diff.") + descDriftDivergent = newDesc("drift_divergent_keys", + "Keys with divergent values per the last full reconciliation diff.") + descDriftOrphan = newDesc("drift_orphan_keys", + "Target keys with no source counterpart per the last full reconciliation diff.") + descLastReconcile = newDesc("last_reconcile_timestamp_seconds", + "Unix time the most recent reconciliation/verification pass completed.") +) + +var allDescs = []*prometheus.Desc{ + descPhase, descWatermark, descSourceRevision, descLag, descSinceProgress, + descKeysApplied, descInitialSyncKeys, descInitialSyncExpected, + descLeaseBackedKeys, descForcedResync, descScanRestart, descResyncLoop, + descThrottled, descQuotaExhausted, descCompacted, descCutoverReady, + descLastErrorClass, descSourceKeys, descTargetKeys, descDriftMissing, + descDriftDivergent, descDriftOrphan, descLastReconcile, +} + +// snapshotCollector adapts Agent.Snapshot to prometheus.Collector: every +// Collect reads one fresh snapshot and emits const metrics — no ticker, no +// staleness window. +type snapshotCollector struct { + snapshot snapshotFn + now func() time.Time +} + +func (c *snapshotCollector) Describe(ch chan<- *prometheus.Desc) { + for _, d := range allDescs { + ch <- d + } +} + +func (c *snapshotCollector) Collect(ch chan<- prometheus.Metric) { + s := c.snapshot() + gauge := func(d *prometheus.Desc, v float64, labels ...string) { + ch <- prometheus.MustNewConstMetric(d, prometheus.GaugeValue, v, labels...) + } + counter := func(d *prometheus.Desc, v float64, labels ...string) { + ch <- prometheus.MustNewConstMetric(d, prometheus.CounterValue, v, labels...) + } + for _, p := range allPhases { + gauge(descPhase, boolGauge(s.Phase == p), string(p)) + } + gauge(descWatermark, float64(s.Watermark)) + gauge(descSourceRevision, float64(s.SourceRevision)) + if s.SourceRevision > 0 { + gauge(descLag, float64(max(s.SourceRevision-s.Watermark, 0))) + } + if !s.LastProgressTime.IsZero() { + gauge(descSinceProgress, c.now().Sub(s.LastProgressTime).Seconds()) + } + counter(descKeysApplied, float64(s.KeysAppliedTotal)) + gauge(descInitialSyncKeys, float64(s.InitialSyncKeyCount)) + gauge(descInitialSyncExpected, float64(s.InitialSyncTotalKeyCount)) + gauge(descLeaseBackedKeys, float64(s.LeaseBackedKeyCount)) + for _, reason := range allResyncReasons { + counter(descForcedResync, float64(s.ForcedResyncCountByReason[reason]), string(reason)) + } + for reason, n := range s.ForcedResyncCountByReason { + if !slices.Contains(allResyncReasons, reason) { // a reason newer than this list + counter(descForcedResync, float64(n), string(reason)) + } + } + counter(descScanRestart, float64(s.ScanRestartCount)) + gauge(descResyncLoop, boolGauge(s.ResyncLoopDetected)) + gauge(descThrottled, boolGauge(s.Throttled)) + gauge(descQuotaExhausted, boolGauge(s.QuotaExhausted)) + gauge(descCompacted, boolGauge(s.Compacted)) + gauge(descCutoverReady, boolGauge(s.CutoverReady)) + for _, class := range allClasses { + gauge(descLastErrorClass, boolGauge(s.LastErrorClass == class), string(class)) + } + gauge(descSourceKeys, float64(s.SourceKeyCount)) + gauge(descTargetKeys, float64(s.TargetKeyCount)) + if d := s.LastReconcileDrift; d != nil { + gauge(descDriftMissing, float64(d.MissingKeys)) + gauge(descDriftDivergent, float64(d.DivergentKeys)) + gauge(descDriftOrphan, float64(d.OrphanKeys)) + } + if !s.LastReconcileTime.IsZero() { + gauge(descLastReconcile, float64(s.LastReconcileTime.Unix())) + } +} + +func boolGauge(b bool) float64 { + if b { + return 1 + } + return 0 +} + +// newRegistry builds the agent's standalone Prometheus registry: process/Go +// runtime collectors, the snapshot collector, and the cert-expiry gauges. +// Deliberately NOT controller-runtime's global registry — that one is +// manager-coupled and the agent is not a manager. +func newRegistry(snapshot snapshotFn, certExpiry *prometheus.GaugeVec) *prometheus.Registry { + reg := prometheus.NewRegistry() + reg.MustRegister( + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + &snapshotCollector{snapshot: snapshot, now: time.Now}, + certExpiry, + ) + return reg +} diff --git a/cmd/mirror-agent/metrics_test.go b/cmd/mirror-agent/metrics_test.go new file mode 100644 index 00000000..e2f6374c --- /dev/null +++ b/cmd/mirror-agent/metrics_test.go @@ -0,0 +1,174 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd-operator/pkg/mirroragent" +) + +func TestSnapshotCollector(t *testing.T) { + snap := fullSnapshot() + c := &snapshotCollector{ + snapshot: func() mirroragent.Snapshot { return snap }, + now: func() time.Time { return snap.LastProgressTime.Add(5 * time.Second) }, + } + reg := prometheus.NewPedanticRegistry() + reg.MustRegister(c) + + // One entry per family: name, type, help (must match metrics.go + // verbatim), samples (metricPrefix is prepended to each). + type family struct { + name, typ, help string + samples []string + } + families := []family{ + {"phase", "gauge", "One-hot agent phase (1 on the current phase).", []string{ + `phase{phase="Connecting"} 0`, + `phase{phase="Degraded"} 0`, + `phase{phase="Drained"} 0`, + `phase{phase="Failed"} 0`, + `phase{phase="InitialSync"} 0`, + `phase{phase="Syncing"} 1`, + }}, + {"watermark_revision", "gauge", + "Checkpoint watermark: the source revision through which the target is caught up.", + []string{"watermark_revision 90"}}, + {"source_revision", "gauge", + "Source cluster revision as of the last watch header.", + []string{"source_revision 100"}}, + {"lag_revisions", "gauge", + "max(0, source_revision - watermark). Overstates lag for prefix-scoped mirrors " + + "(the source revision is cluster-global). Absent until the source revision is known.", + []string{"lag_revisions 10"}}, + {"seconds_since_last_progress", "gauge", + "Seconds since the watermark last advanced. Absent until first progress.", + []string{"seconds_since_last_progress 5"}}, + {"keys_applied_total", "counter", + "Data operations (puts and deletes) committed to the target in fenced Txns.", + []string{"keys_applied_total 123"}}, + {"initial_sync_keys", "gauge", + "Keys copied so far by the genesis scan.", + []string{"initial_sync_keys 40"}}, + {"initial_sync_expected_keys", "gauge", + "Expected key total for the genesis scan (the progress denominator).", + []string{"initial_sync_expected_keys 50"}}, + {"lease_backed_keys", "gauge", + "Mirrored keys whose source copy is lease-backed.", + []string{"lease_backed_keys 3"}}, + {"forced_resync_total", "counter", + "Forced resyncs by trigger reason.", + []string{ + `forced_resync_total{reason="ClusterIDMismatch"} 1`, + `forced_resync_total{reason="Compacted"} 2`, + }}, + {"scan_restart_total", "counter", + "Genesis scan attempts aborted and restarted from a fresh R0 " + + "(cause in /statusz lastScanRestartCause).", + []string{"scan_restart_total 4"}}, + {"resync_loop_detected", "gauge", + "1 while the resync-loop livelock detector is latched.", + []string{"resync_loop_detected 1"}}, + {"throttled", "gauge", + "1 while the target is rejecting the write rate (backoff engaged).", + []string{"throttled 1"}}, + {"quota_exhausted", "gauge", + "1 while the target reports NOSPACE (parked on the quota probe).", + []string{"quota_exhausted 0"}}, + {"compacted", "gauge", + "1 while recovering from source compaction outrunning the watch.", + []string{"compacted 1"}}, + {"cutover_ready", "gauge", + "1 once the drain completed and the fence role is Primary.", + []string{"cutover_ready 1"}}, + {"last_error_class", "gauge", + "One-hot class of the most recent failure (all 0 when the last attempt succeeded).", + []string{ + `last_error_class{class="Permanent"} 0`, + `last_error_class{class="Quota"} 0`, + `last_error_class{class="Resync"} 0`, + `last_error_class{class="Throttle"} 1`, + `last_error_class{class="Transient"} 0`, + }}, + {"source_keys", "gauge", + "In-scope source key count from the most recent reconciliation/verification pass.", + []string{"source_keys 10"}}, + {"target_keys", "gauge", + "In-scope target key count from the most recent reconciliation/verification pass.", + []string{"target_keys 9"}}, + {"drift_missing_keys", "gauge", + "Keys missing on the target per the last full reconciliation diff.", + []string{"drift_missing_keys 1"}}, + {"drift_divergent_keys", "gauge", + "Keys with divergent values per the last full reconciliation diff.", + []string{"drift_divergent_keys 2"}}, + {"drift_orphan_keys", "gauge", + "Target keys with no source counterpart per the last full reconciliation diff.", + []string{"drift_orphan_keys 3"}}, + {"last_reconcile_timestamp_seconds", "gauge", + "Unix time the most recent reconciliation/verification pass completed.", + []string{fmt.Sprintf("last_reconcile_timestamp_seconds %d", snap.LastReconcileTime.Unix())}}, + } + var expected strings.Builder + for _, f := range families { + fmt.Fprintf(&expected, "# HELP %[1]s%[2]s %[3]s\n# TYPE %[1]s%[2]s %[4]s\n", + metricPrefix, f.name, f.help, f.typ) + for _, s := range f.samples { + expected.WriteString(metricPrefix + s + "\n") + } + } + + require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(expected.String()))) +} + +// TestSnapshotCollectorConditionalAbsence pins which families are omitted +// while their inputs are unknown: lag before the source revision is probed, +// progress age before first progress, drift gauges before a full diff, and +// the reconcile timestamp before any pass. +func TestSnapshotCollectorConditionalAbsence(t *testing.T) { + c := &snapshotCollector{ + snapshot: func() mirroragent.Snapshot { return mirroragent.Snapshot{} }, + now: time.Now, + } + require.NoError(t, testutil.CollectAndCompare(c, strings.NewReader(""), + metricPrefix+"lag_revisions", + metricPrefix+"seconds_since_last_progress", + metricPrefix+"drift_missing_keys", + metricPrefix+"drift_divergent_keys", + metricPrefix+"drift_orphan_keys", + metricPrefix+"last_reconcile_timestamp_seconds", + )) + // The one-hots are always fully emitted, even before Run sets a phase. + require.Equal(t, 6, testutil.CollectAndCount(c, metricPrefix+"phase")) + require.Equal(t, 5, testutil.CollectAndCount(c, metricPrefix+"last_error_class")) + // forced_resync_total is born at 0 for every known reason so increase() + // can see the first resync (a counter's first sample yields no delta). + require.NoError(t, testutil.CollectAndCompare(c, strings.NewReader(fmt.Sprintf(` +# HELP %[1]sforced_resync_total Forced resyncs by trigger reason. +# TYPE %[1]sforced_resync_total counter +%[1]sforced_resync_total{reason="ClusterIDMismatch"} 0 +%[1]sforced_resync_total{reason="Compacted"} 0 +`, metricPrefix)), metricPrefix+"forced_resync_total")) +} diff --git a/cmd/mirror-agent/server.go b/cmd/mirror-agent/server.go new file mode 100644 index 00000000..53121bc0 --- /dev/null +++ b/cmd/mirror-agent/server.go @@ -0,0 +1,88 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "go.etcd.io/etcd-operator/pkg/mirroragent" +) + +// snapshotFn supplies the engine snapshot the HTTP surface serves +// (Agent.Snapshot in production, stubs in tests). +type snapshotFn func() mirroragent.Snapshot + +// readHeaderTimeout bounds header reads on the local listener. Plaintext +// net/http never negotiates HTTP/2, consistent with the operator's h2-off +// stance. +const readHeaderTimeout = 10 * time.Second + +// newMux serves the agent's observability surface: +// +// - /statusz: JSON dump of the engine Snapshot. The tagged +// mirroragent.Snapshot IS the wire shape — the controller decodes into +// the same Go type, so the surface cannot drift from the engine. +// - /healthz: pure process liveness, always 200. A Degraded/backing-off +// agent must not be killed by the kubelet — backoff is normal operation. +// - /readyz: 200 once the engine has connected and owns (or completed) +// the replication loop; see readyPhase for the exact semantics. +// - /metrics: Prometheus exposition over the agent's standalone registry. +func newMux(snapshot snapshotFn, reg *prometheus.Registry) *http.ServeMux { + mux := http.NewServeMux() + mux.HandleFunc("/statusz", func(w http.ResponseWriter, _ *http.Request) { + body, err := json.Marshal(snapshot()) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + _, _ = w.Write(body) + }) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("ok")) + }) + mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + if p := snapshot().Phase; !readyPhase(p) { + http.Error(w, fmt.Sprintf("not ready: phase %q", p), http.StatusServiceUnavailable) + return + } + _, _ = w.Write([]byte("ok")) + }) + mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) + return mux +} + +// readyPhase defines /readyz: ready = "the engine has connected and owns +// (or completed) the replication loop". Degraded IS ready — transient +// backoff must not flap the Deployment's availability. Drained IS ready — +// terminal success. Connecting ("" before Run starts) is not, gating +// rollout until both sides are reachable; Failed is not — permanent. +func readyPhase(p mirroragent.Phase) bool { + switch p { + case "", mirroragent.PhaseConnecting, mirroragent.PhaseFailed: + return false + default: + return true + } +} diff --git a/cmd/mirror-agent/server_test.go b/cmd/mirror-agent/server_test.go new file mode 100644 index 00000000..f2f2b080 --- /dev/null +++ b/cmd/mirror-agent/server_test.go @@ -0,0 +1,128 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd-operator/pkg/mirroragent" +) + +// fullSnapshot fabricates a Snapshot with every field populated. Fixed UTC +// times (no monotonic clock) so JSON round-trips compare equal. +func fullSnapshot() mirroragent.Snapshot { + at := func(min int) time.Time { return time.Date(2026, 7, 7, 12, min, 0, 0, time.UTC) } + return mirroragent.Snapshot{ + Phase: mirroragent.PhaseSyncing, + SourceVersion: "3.5.9", + TargetVersion: "3.6.0", + SourceClusterID: 11, + TargetClusterID: 22, + Watermark: 90, + SourceRevision: 100, + LastProgressTime: at(1), + InitialSyncKeyCount: 40, + InitialSyncTotalKeyCount: 50, + InitialSyncStartTime: at(2), + InitialSyncCompletionTime: at(3), + KeysAppliedTotal: 123, + LeaseBackedKeyCount: 3, + ForcedResyncCount: 3, + ForcedResyncCountByReason: map[mirroragent.ResyncReason]int64{ + mirroragent.ResyncReasonCompacted: 2, + mirroragent.ResyncReasonClusterIDMismatch: 1, + }, + LastResyncReason: mirroragent.ResyncReasonCompacted, + ResyncLoopDetected: true, + ScanRestartCount: 4, + LastScanRestartCause: mirroragent.ScanRestartWatchBufferOverflow, + SourceKeyCount: 10, + TargetKeyCount: 9, + Throttled: true, + QuotaExhausted: false, + Compacted: true, + LastReconcileTime: at(4), + LastReconcileDrift: &mirroragent.Drift{MissingKeys: 1, DivergentKeys: 2, OrphanKeys: 3, Repaired: true}, + LastError: "boom", + LastErrorClass: mirroragent.ClassThrottle, + CutoverReady: true, + Cutover: &mirroragent.CutoverStatus{ + DrainTargetRevision: 100, DrainedRevision: 100, VerifiedTime: at(5), + SourceKeyCount: 10, TargetKeyCount: 10, LeasedKeyCount: 3, + }, + } +} + +func get(t *testing.T, mux *http.ServeMux, path string) *httptest.ResponseRecorder { + t.Helper() + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + return rec +} + +func TestStatuszHandler(t *testing.T) { + want := fullSnapshot() + mux := newMux(func() mirroragent.Snapshot { return want }, prometheus.NewRegistry()) + + rec := get(t, mux, "/statusz") + require.Equal(t, http.StatusOK, rec.Code) + require.Equal(t, "application/json", rec.Header().Get("Content-Type")) + require.Equal(t, "no-store", rec.Header().Get("Cache-Control")) + + var got mirroragent.Snapshot + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + require.Equal(t, want, got, "the wire shape round-trips into mirroragent.Snapshot losslessly") +} + +func TestReadyzPhases(t *testing.T) { + cases := []struct { + phase mirroragent.Phase + want int + }{ + {"", http.StatusServiceUnavailable}, + {mirroragent.PhaseConnecting, http.StatusServiceUnavailable}, + {mirroragent.PhaseFailed, http.StatusServiceUnavailable}, + {mirroragent.PhaseInitialSync, http.StatusOK}, + {mirroragent.PhaseSyncing, http.StatusOK}, + {mirroragent.PhaseDegraded, http.StatusOK}, + {mirroragent.PhaseDrained, http.StatusOK}, + } + for _, tc := range cases { + t.Run(string(tc.phase), func(t *testing.T) { + mux := newMux(func() mirroragent.Snapshot { + return mirroragent.Snapshot{Phase: tc.phase} + }, prometheus.NewRegistry()) + require.Equal(t, tc.want, get(t, mux, "/readyz").Code) + }) + } +} + +func TestHealthzAlwaysOK(t *testing.T) { + mux := newMux(func() mirroragent.Snapshot { + return mirroragent.Snapshot{Phase: mirroragent.PhaseFailed} + }, prometheus.NewRegistry()) + rec := get(t, mux, "/healthz") + require.Equal(t, http.StatusOK, rec.Code) + require.Equal(t, "ok", rec.Body.String()) +} diff --git a/cmd/mirror-agent/tlsinfo.go b/cmd/mirror-agent/tlsinfo.go new file mode 100644 index 00000000..e913fbf5 --- /dev/null +++ b/cmd/mirror-agent/tlsinfo.go @@ -0,0 +1,87 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "strings" + + "go.etcd.io/etcd/client/pkg/v3/transport" +) + +// buildTLSInfo translates one side's TLS flags into a transport.TLSInfo; +// enabled is false when ---tls is off (dial cleartext, nil tls.Config). +// +// ROTATION: TLSInfo.ClientConfig installs a per-handshake +// GetClientCertificate that re-reads the leaf pair from the mounted paths, +// so client certificate rotation needs no restart — the contract pinned on +// EtcdMirrorTLS. The CA pool from TrustedCAFile is loaded once at client +// construction; CA rotation relies on the standard overlapping-bundle +// practice, which is what the separate ---ca-bundle-file exists for. +// +// A fully empty TLSInfo (tls on, no file flags) yields server-auth TLS +// verified against the system trust roots, matching a tls block with a nil +// secretRef in the CRD. +func buildTLSInfo(side *sideFlags) (transport.TLSInfo, bool, error) { + if !side.tls { + // TLS material without ---tls would be silently dropped and the + // client would dial cleartext; fail fast instead, like the https:// + // endpoint scheme check. + if flags := tlsMaterialFlags(side); len(flags) > 0 { + return transport.TLSInfo{}, false, fmt.Errorf( + "%s requires --%s-tls", strings.Join(flags, ", "), side.side) + } + return transport.TLSInfo{}, false, nil + } + if (side.certFile == "") != (side.keyFile == "") { + return transport.TLSInfo{}, false, fmt.Errorf( + "--%s-cert-file and --%s-key-file must be set together", side.side, side.side) + } + trust := side.caFile + if side.caBundleFile != "" { + if side.caFile != "" { + setupLog.Info("CA bundle takes precedence over the CA file", + "side", side.side, "caBundleFile", side.caBundleFile, "caFile", side.caFile) + } + trust = side.caBundleFile + } + return transport.TLSInfo{ + CertFile: side.certFile, + KeyFile: side.keyFile, + TrustedCAFile: trust, + ServerName: side.serverName, + InsecureSkipVerify: side.insecureSkipVerify, + }, true, nil +} + +// tlsMaterialFlags names the TLS flags a side has set that are meaningless +// without ---tls. +func tlsMaterialFlags(side *sideFlags) []string { + var out []string + add := func(set bool, name string) { + if set { + out = append(out, "--"+side.side+"-"+name) + } + } + add(side.certFile != "", "cert-file") + add(side.keyFile != "", "key-file") + add(side.caFile != "", "ca-file") + add(side.caBundleFile != "", "ca-bundle-file") + add(side.serverName != "", "server-name") + add(side.insecureSkipVerify, "insecure-skip-verify") + return out +} diff --git a/cmd/mirror-agent/tlsinfo_test.go b/cmd/mirror-agent/tlsinfo_test.go new file mode 100644 index 00000000..f51208b6 --- /dev/null +++ b/cmd/mirror-agent/tlsinfo_test.go @@ -0,0 +1,89 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBuildTLSInfo(t *testing.T) { + t.Run("disabled", func(t *testing.T) { + info, enabled, err := buildTLSInfo(&sideFlags{side: sideSource}) + require.NoError(t, err) + require.False(t, enabled, "no --source-tls means cleartext (nil tls.Config)") + require.True(t, info.Empty()) + }) + + t.Run("enabled with no files uses system roots", func(t *testing.T) { + info, enabled, err := buildTLSInfo(&sideFlags{side: sideSource, tls: true}) + require.NoError(t, err) + require.True(t, enabled) + require.True(t, info.Empty()) + require.Empty(t, info.TrustedCAFile) + }) + + t.Run("bundle takes precedence over ca file", func(t *testing.T) { + info, enabled, err := buildTLSInfo(&sideFlags{ + side: sideTarget, tls: true, + caFile: "/mnt/tls/ca.crt", caBundleFile: "/mnt/bundle/ca.crt", + }) + require.NoError(t, err) + require.True(t, enabled) + require.Equal(t, "/mnt/bundle/ca.crt", info.TrustedCAFile) + }) + + t.Run("propagates server name, skip-verify and leaf paths", func(t *testing.T) { + info, _, err := buildTLSInfo(&sideFlags{ + side: sideSource, tls: true, + certFile: "/mnt/tls/tls.crt", keyFile: "/mnt/tls/tls.key", + caFile: "/mnt/tls/ca.crt", serverName: "etcd.example.com", + insecureSkipVerify: true, + }) + require.NoError(t, err) + require.Equal(t, "/mnt/tls/tls.crt", info.CertFile) + require.Equal(t, "/mnt/tls/tls.key", info.KeyFile) + require.Equal(t, "/mnt/tls/ca.crt", info.TrustedCAFile) + require.Equal(t, "etcd.example.com", info.ServerName) + require.True(t, info.InsecureSkipVerify) + }) + + t.Run("cert without key", func(t *testing.T) { + _, _, err := buildTLSInfo(&sideFlags{side: sideSource, tls: true, certFile: "/mnt/tls/tls.crt"}) + require.ErrorContains(t, err, "must be set together") + }) + + // TLS material without ---tls is a hard error, never a silent + // cleartext downgrade. + t.Run("material flags require tls", func(t *testing.T) { + _, _, err := buildTLSInfo(&sideFlags{ + side: sideSource, + certFile: "/mnt/tls/tls.crt", keyFile: "/mnt/tls/tls.key", caFile: "/mnt/tls/ca.crt", + }) + require.ErrorContains(t, err, "--source-cert-file, --source-key-file, --source-ca-file requires --source-tls") + + for _, side := range []*sideFlags{ + {side: sideTarget, caBundleFile: "/mnt/bundle/ca.crt"}, + {side: sideTarget, serverName: "etcd.example.com"}, + {side: sideTarget, insecureSkipVerify: true}, + } { + _, _, err := buildTLSInfo(side) + require.ErrorContains(t, err, "requires --target-tls") + } + }) +} diff --git a/config/crd/bases/operator.etcd.io_etcdclusters.yaml b/config/crd/bases/operator.etcd.io_etcdclusters.yaml index 0ae1e80e..e92496c1 100644 --- a/config/crd/bases/operator.etcd.io_etcdclusters.yaml +++ b/config/crd/bases/operator.etcd.io_etcdclusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.21.0 name: etcdclusters.operator.etcd.io spec: group: operator.etcd.io diff --git a/config/crd/bases/operator.etcd.io_etcdmirrors.yaml b/config/crd/bases/operator.etcd.io_etcdmirrors.yaml new file mode 100644 index 00000000..e713af47 --- /dev/null +++ b/config/crd/bases/operator.etcd.io_etcdmirrors.yaml @@ -0,0 +1,2060 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: etcdmirrors.operator.etcd.io +spec: + group: operator.etcd.io + names: + kind: EtcdMirror + listKind: EtcdMirrorList + plural: etcdmirrors + singular: etcdmirror + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Available")].status + name: Available + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.lastAppliedRevision + name: Revision + type: integer + - jsonPath: .status.sourceRevision + name: Source-Rev + type: integer + - jsonPath: .status.lastProgressTime + name: Last-Progress + type: date + - jsonPath: .spec.source.prefix + name: Source + priority: 1 + type: string + - jsonPath: .spec.target.prefix + name: Target + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + EtcdMirror is the Schema for the etcdmirrors API. It describes a + continuous, one-way key-range sync from a source etcd cluster to a target + etcd cluster, run as a single supervised stateless pod (a size-1 + Deployment; progress lives in a fenced checkpoint key in the target etcd, + not on a volume). + + It is a byte-copy of keys and values, not a replica: revisions, versions, + and create/mod ordering are target-assigned, and leases are stripped. See + Fidelity Caveats in docs/etcdmirror.md before depending on anything but + key/value content. + + Two-way sync: never — etcd revisions are cluster-local and there is no + per-key provenance channel, so bidirectional sync is structurally + inexpressible. Reversal for cutover/failback: yes — delete the CR and + create a new one with swapped endpoints, initialSync.mode + OverwriteAndPrune, only after the forward mirror reported CutoverReady. + See docs/etcdmirror.md. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + EtcdMirrorSpec defines the desired state of an EtcdMirror. + + Range-defining and rewrite fields are immutable (CEL transition rules + below): source.prefix, target.prefix, sync.destPrefix, sync.excludePrefixes + and checkpoint.key. Changing what range is mirrored, or where it lands, + mid-life silently diverges: a restarted agent resumes from its checkpoint + without a scan, so removing an exclusion never backfills pre-existing keys + and adding one strands already-mirrored keys as permanent orphans — all + with every condition green. Endpoints stay mutable — rotating an NLB DNS + name or adding a member to the same cluster is routine; pointing at a + different cluster is caught at runtime by the checkpoint's dual-cluster-ID + binding, not by spec validation. + + The transition rules compare VALUES, presence-normalized: for these + fields the empty value and an absent field are semantically identical + (prefix "" = whole keyspace, destPrefix "" = strip the source prefix), and + Go typed clients drop explicit "" through omitempty — presence-based rules + would reject every typed-client update of a CR created with an explicit + empty string. + properties: + checkpoint: + description: Checkpoint configures the reserved checkpoint/fence key + on the target. + properties: + key: + description: |- + Key overrides the reserved checkpoint key. Defaults to the effective + destination prefix + "\x00etcdmirror-checkpoint" — the \x00 byte after + the prefix cannot collide with any real key under it. MUST live under + the effective destination prefix (target.prefix + sync.destPrefix, + CEL-enforced): the range-scoped target credential covers it, and the + exact-match exclusion from scans/counts/prune only works inside the + mirrored range. Immutable after creation. + type: string + type: object + initialSync: + description: |- + InitialSync governs genesis behavior: how pre-existing destination keys + are treated (Mode) and optionally where replication starts + (StartRevision). + properties: + mode: + default: RequireEmpty + description: |- + Mode governs pre-existing destination keys at genesis. Defaults to + RequireEmpty (refuse a non-empty destination prefix). + enum: + - RequireEmpty + - Overwrite + - OverwriteAndPrune + type: string + startRevision: + description: |- + StartRevision, when > 0, skips the genesis scan entirely and starts + watching from StartRevision+1. For fidelity-preserving seeds: restore + the target from a source snapshot (`etcdutl snapshot restore + --bump-revision --mark-compacted`), then mirror only the delta. + Requires Mode Overwrite or OverwriteAndPrune (CEL-enforced). + format: int64 + minimum: 0 + type: integer + type: object + x-kubernetes-validations: + - message: initialSync.startRevision requires initialSync.mode Overwrite + or OverwriteAndPrune (a seeded target is not empty) + rule: '!(has(self.startRevision) && self.startRevision > 0 && (!has(self.mode) + || self.mode == ''RequireEmpty''))' + mode: + default: Sync + description: |- + Mode selects continuous replication (Sync, the default) or a cutover + drain (Drain). See EtcdMirrorModeDrain for the cutover contract. + enum: + - Sync + - Drain + type: string + paused: + description: |- + Paused, when true, scales the agent Deployment to zero without deleting + the CR or its checkpoint. The checkpoint lives in the target etcd, so + resume picks up from the last fenced watermark. NOTE: pausing longer + than the source's compaction retention guarantees a full forced resync + on resume — there is no free lunch past the retention window. + type: boolean + podTemplate: + description: |- + PodTemplate carries scheduling/affinity/labels/annotations for the + agent pod, reusing EtcdClusterSpec's PodTemplate shape verbatim. + properties: + metadata: + description: Metadata is the metadata to add to the pod. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + affinity: + description: Affinity is a group of affinity scheduling rules. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules + for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, in the + range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + type: object + reconciliation: + description: |- + Reconciliation optionally enables a periodic full diff-and-repair pass + layered on top of the continuous watch-based mirror. Independent of + this setting, one reconciliation-with-delete pass always runs after any + forced resync (mark-and-sweep), and as the OverwriteAndPrune genesis + pass and the Drain verification pass. + properties: + deleteOrphans: + description: |- + DeleteOrphans, when true, allows the PERIODIC pass to delete target + keys under the destination prefix that have no corresponding source + key. Defaults to false. (Forced-resync sweeps and OverwriteAndPrune + always delete orphans; this knob only governs the periodic pass.) + type: boolean + enabled: + description: |- + Enabled toggles the PERIODIC pass. Defaults to false: it is a full + diff of the prefix contents on both sides (O(keyspace)), so it is + opt-in. + type: boolean + interval: + description: Interval between periodic passes. Defaults to 1h + when Enabled. + type: string + type: object + resources: + description: |- + Resources are the agent container's compute resources. The agent's + memory model is bounded by Sync.PageBytes (single in-flight scan page, + no unbounded read-ahead); size limits accordingly. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + source: + description: |- + Source is the etcd cluster keys are read from. EtcdMirror never writes + back to Source; the agent's source-side client is only ever used for + Get/Watch, never Put/Delete/Txn. + + VERSION FLOOR: source etcd must be >= 3.4 (probed via maintenance + Status() at connect; below the floor the mirror goes Failed with reason + UnsupportedVersion). >= 3.4.25 / 3.5.8 is the recommended floor: below + it, watch progress notifications are unreliable and the agent cannot + trust the watermark machinery that drives lag, the checkpoint, and the + Drain gate. + properties: + auth: + description: Auth configures etcd username/password (RBAC) auth + for THIS side. + properties: + secretRef: + description: |- + SecretRef names a Secret holding "username" and "password" keys. If + this whole Auth block is nil, the agent does not call etcd's + Authenticate() at all; the pinned v3 client transparently re-auths on + token expiry. PRECEDENCE: when both a client certificate and Auth are + supplied, etcd uses the token identity, not the certificate CN — the + Auth user must hold the range-scoped role. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - secretRef + type: object + x-kubernetes-validations: + - message: secretRef.name is required + rule: has(self.secretRef.name) && size(self.secretRef.name) + > 0 + endpointList: + description: |- + EndpointList is a raw set of etcd client-URL host:port (or + scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379". + This is the ONLY supported mechanism for a cluster external to this + Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB); + there is deliberately no tunnel/port-forward mode — terminate any + tunnel upstream and hand EtcdMirror the resulting stable endpoint(s). + + Prefer listing per-member endpoints over a single load-balancer VIP: a + TCP-health-checked VIP cannot see etcd quorum, and the client's own + balancer handles per-member failover. + + IP-LITERAL ENDPOINTS: Go's TLS stack requires an IP SAN (not a DNS SAN) + to verify a bare-IP endpoint. Either set EtcdMirrorTLS.ServerName to a + hostname present as a DNS SAN on the certificate, or ensure the + certificate carries a matching IP SAN. Fix the SAN/ServerName mismatch; + do not reach for InsecureSkipVerify. + items: + type: string + type: array + prefix: + description: |- + Prefix is the etcd key prefix on THIS side. On Source, only keys under + this prefix are synced; empty means the whole keyspace. On Target, this + is the prefix under which mirrored keys land (see EtcdMirrorSyncSpec's + rewrite formula). Immutable after creation. + type: string + serviceRef: + description: |- + ServiceRef points at a Kubernetes Service in this cluster whose DNS + name resolves the etcd client endpoint(s), for the same-cluster case. + Namespace defaults to the EtcdMirror's own namespace when empty. + properties: + name: + description: Name is the Service name. + minLength: 1 + type: string + namespace: + description: Namespace defaults to the EtcdMirror's own namespace + when empty. + type: string + port: + description: |- + Port is the Service port name or number exposing etcd's client API. + Defaults to "client" when empty, matching PodMonitorSpec.Port's + convention elsewhere in this API group. + type: string + required: + - name + type: object + tls: + description: |- + TLS configures the agent's client TLS for THIS side. Nil means the + agent dials this side in cleartext (and https:// endpoints are + CEL-rejected). An empty block means server-auth TLS verified against + the system trust roots. + properties: + caBundleRef: + description: |- + CABundleRef optionally sources the trust anchors from a separate + Secret or ConfigMap key, decoupling trust from the identity Secret + (Gateway API caCertificateRefs precedent). Takes precedence over + SecretRef's ca.crt. + properties: + key: + description: Key within the object. Defaults to "ca.crt". + type: string + kind: + default: ConfigMap + description: Kind is Secret or ConfigMap. Defaults to + ConfigMap. + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the Secret or ConfigMap, in the EtcdMirror's + namespace. + minLength: 1 + type: string + required: + - name + type: object + insecureSkipVerify: + default: false + description: |- + InsecureSkipVerify disables server certificate verification. Strongly + discouraged, especially for a source reached over the public internet. + Requires InsecureSkipVerifyAcknowledgeRisk to also be set true + (CEL-enforced). The controller additionally emits a standing Warning + event whenever this is true. + type: boolean + insecureSkipVerifyAcknowledgeRisk: + default: false + description: |- + InsecureSkipVerifyAcknowledgeRisk must independently be set true + whenever InsecureSkipVerify is true (CEL-enforced companion field). Its + only purpose is to require a deliberate, separate, reviewable line in + the manifest diff before disabling TLS verification. + type: boolean + secretRef: + description: |- + SecretRef names a Secret (in the EtcdMirror's namespace) holding this + side's TLS material in the standard kubernetes.io/tls-compatible shape: + - ca.crt: PEM CA bundle used to verify the peer's server certificate + (unless CABundleRef overrides it, or InsecureSkipVerify is true). + - tls.crt / tls.key: PEM client certificate + key, for mTLS. + Optional — omit both for server-auth-only TLS. + Nil means no client identity and verification against the system trust + roots (the etcdctl default). Note a source running with + --client-cert-auth (the RKE1 default) rejects certless clients at the + handshake regardless of etcd RBAC auth; server-auth-only + Auth is not + viable against such a source. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + serverName: + description: |- + ServerName overrides the TLS ServerName (SNI) used for verification, + for cases where the dialed address doesn't match a SAN on the + certificate (e.g. dialing an NLB IP directly). Applies to EVERY + endpoint in the list, so mixing endpoints with different certificates + behind one ServerName will fail verification on the mismatched ones. + type: string + type: object + x-kubernetes-validations: + - message: insecureSkipVerify requires insecureSkipVerifyAcknowledgeRisk + to also be true + rule: '!self.insecureSkipVerify || self.insecureSkipVerifyAcknowledgeRisk' + - message: secretRef.name must be non-empty when secretRef is + set + rule: '!has(self.secretRef) || (has(self.secretRef.name) && + size(self.secretRef.name) > 0)' + type: object + x-kubernetes-validations: + - message: exactly one of endpointList or serviceRef must be set + rule: (has(self.endpointList) && size(self.endpointList) > 0) != + has(self.serviceRef) + - message: 'http:// endpoints conflict with a tls block: use https:// + endpoints or remove tls' + rule: '!(has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, + e.startsWith(''http://'')))' + - message: https:// endpoints require a tls block (an empty tls block + selects server-auth TLS against system trust roots) + rule: '!(!has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, + e.startsWith(''https://'')))' + sync: + description: |- + Sync tunes runtime sync behavior (batching, paging, rate limiting, + prefix rewrite, timeouts, backoff). + properties: + destPrefix: + description: |- + DestPrefix is the middle term of the rewrite formula above. Default "" + means the source prefix is stripped and key remainders land directly + under target.prefix. Immutable after creation. + type: string + dialTimeout: + description: |- + DialTimeout bounds establishing the initial client connection to each + side. Defaults to 10s. + type: string + excludePrefixes: + description: |- + ExcludePrefixes lists source key prefixes (full source-side keys, e.g. + "/registry/events/") skipped entirely: not scanned, not watched, not + counted, not pruned. Use to drop high-churn low-value ranges and cut + WAN cost, or to skip lease-backed ranges that don't survive mirroring. + Nested/duplicate entries are normalized by the agent (an entry covered + by another is dropped). Immutable after creation (it defines the + mirrored range): removing an exclusion would require a backfill scan + the checkpoint-resume path never runs, and adding one would strand + already-mirrored keys as permanent orphans — change it via + delete-and-recreate with an appropriate initialSync.mode instead. + items: + type: string + maxItems: 64 + type: array + maxOpsPerSecond: + description: |- + MaxOpsPerSecond rate-limits the agent's target write rate (a token + bucket over puts+deletes/sec), applied to both the genesis scan and + watch-driven applies. Zero (default) means unlimited. Mind the + retention prerequisite above when throttling. + format: int32 + minimum: 0 + type: integer + maxTxnOps: + description: |- + MaxTxnOps bounds how many operations the agent batches into a single + target Txn, including the reserved checkpoint-write slot. Must not + exceed the target's --max-txn-ops (etcd default 128). Defaults to 128. + format: int32 + minimum: 2 + type: integer + pageBytes: + anyOf: + - type: integer + - type: string + description: PageBytes bounds bytes per source scan page. Defaults + to 1Mi. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + pageKeyLimit: + description: |- + PageKeyLimit bounds keys per source scan page during InitialSync and + reconciliation. The scan is pull-based, one page in flight — no + read-ahead — so this and PageBytes bound agent memory. Defaults to 512. + format: int32 + minimum: 1 + type: integer + reconnectBackoff: + description: |- + ReconnectBackoff bounds the retry/backoff loop wrapping connection-class + errors. Throttling-class errors (target rate rejection) use a more + conservative curve derived from the same bounds; quota exhaustion + (TargetQuotaExhausted) and permanent errors are never retried through + this loop. Defaults to exponential backoff from 1s to 30s. + properties: + initialDelay: + type: string + maxDelay: + type: string + type: object + requestTimeout: + description: |- + RequestTimeout is the per-RPC context deadline applied to every unary + call on both sides (watches excluded — they are long-lived by design + and covered by progress-notification liveness instead). Without it a + blackholed call through an NLB never errors and backoff never engages. + Defaults to 30s. + type: string + txnFlushBytes: + anyOf: + - type: integer + - type: string + description: |- + TxnFlushBytes is the byte watermark at which a batch is flushed (at the + next source-revision boundary). Keep well under etcd's request size + limits: a Txn over ~1.5MiB is rejected by the server and one over 2MiB + by the client send cap — both classified permanent errors, not + throttling. Defaults to 1Mi. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + watchBufferBytes: + anyOf: + - type: integer + - type: string + description: |- + WatchBufferBytes bounds the memory used to buffer watch events + observed from R0 while the genesis scan runs (the reflector replay + buffer). On overflow the agent cancels the source watch and restarts + the scan from a fresh R0 (see the InitialSyncCompactionRaced event) — + a bounded retry instead of unbounded growth when source churn outruns + scan+apply throughput. Defaults to 16Mi (must stay in lockstep with + pkg/mirroragent's DefaultWatchBufferBytes). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + target: + description: |- + Target is the etcd cluster keys are written into. + + SECURITY PREREQUISITE: Target's credential (etcd RBAC user and/or the + client certificate's role) MUST be range-scoped to the effective + destination prefix — never a cluster-admin-equivalent credential. The + agent's client-side rewrite logic is defense against bugs in its own + code, NOT a security boundary. The grant must also cover the reserved + checkpoint key (see Checkpoint). Configure via `etcdctl role + grant-permission --prefix=true readwrite ` before + pointing an EtcdMirror at the cluster. + + The target must run with auto-compaction enabled: forced-resync churn + and prune passes march an uncompacted target toward its storage quota + (2GiB by default), which surfaces as TargetQuotaExhausted. + properties: + auth: + description: Auth configures etcd username/password (RBAC) auth + for THIS side. + properties: + secretRef: + description: |- + SecretRef names a Secret holding "username" and "password" keys. If + this whole Auth block is nil, the agent does not call etcd's + Authenticate() at all; the pinned v3 client transparently re-auths on + token expiry. PRECEDENCE: when both a client certificate and Auth are + supplied, etcd uses the token identity, not the certificate CN — the + Auth user must hold the range-scoped role. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - secretRef + type: object + x-kubernetes-validations: + - message: secretRef.name is required + rule: has(self.secretRef.name) && size(self.secretRef.name) + > 0 + endpointList: + description: |- + EndpointList is a raw set of etcd client-URL host:port (or + scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379". + This is the ONLY supported mechanism for a cluster external to this + Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB); + there is deliberately no tunnel/port-forward mode — terminate any + tunnel upstream and hand EtcdMirror the resulting stable endpoint(s). + + Prefer listing per-member endpoints over a single load-balancer VIP: a + TCP-health-checked VIP cannot see etcd quorum, and the client's own + balancer handles per-member failover. + + IP-LITERAL ENDPOINTS: Go's TLS stack requires an IP SAN (not a DNS SAN) + to verify a bare-IP endpoint. Either set EtcdMirrorTLS.ServerName to a + hostname present as a DNS SAN on the certificate, or ensure the + certificate carries a matching IP SAN. Fix the SAN/ServerName mismatch; + do not reach for InsecureSkipVerify. + items: + type: string + type: array + prefix: + description: |- + Prefix is the etcd key prefix on THIS side. On Source, only keys under + this prefix are synced; empty means the whole keyspace. On Target, this + is the prefix under which mirrored keys land (see EtcdMirrorSyncSpec's + rewrite formula). Immutable after creation. + type: string + serviceRef: + description: |- + ServiceRef points at a Kubernetes Service in this cluster whose DNS + name resolves the etcd client endpoint(s), for the same-cluster case. + Namespace defaults to the EtcdMirror's own namespace when empty. + properties: + name: + description: Name is the Service name. + minLength: 1 + type: string + namespace: + description: Namespace defaults to the EtcdMirror's own namespace + when empty. + type: string + port: + description: |- + Port is the Service port name or number exposing etcd's client API. + Defaults to "client" when empty, matching PodMonitorSpec.Port's + convention elsewhere in this API group. + type: string + required: + - name + type: object + tls: + description: |- + TLS configures the agent's client TLS for THIS side. Nil means the + agent dials this side in cleartext (and https:// endpoints are + CEL-rejected). An empty block means server-auth TLS verified against + the system trust roots. + properties: + caBundleRef: + description: |- + CABundleRef optionally sources the trust anchors from a separate + Secret or ConfigMap key, decoupling trust from the identity Secret + (Gateway API caCertificateRefs precedent). Takes precedence over + SecretRef's ca.crt. + properties: + key: + description: Key within the object. Defaults to "ca.crt". + type: string + kind: + default: ConfigMap + description: Kind is Secret or ConfigMap. Defaults to + ConfigMap. + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the Secret or ConfigMap, in the EtcdMirror's + namespace. + minLength: 1 + type: string + required: + - name + type: object + insecureSkipVerify: + default: false + description: |- + InsecureSkipVerify disables server certificate verification. Strongly + discouraged, especially for a source reached over the public internet. + Requires InsecureSkipVerifyAcknowledgeRisk to also be set true + (CEL-enforced). The controller additionally emits a standing Warning + event whenever this is true. + type: boolean + insecureSkipVerifyAcknowledgeRisk: + default: false + description: |- + InsecureSkipVerifyAcknowledgeRisk must independently be set true + whenever InsecureSkipVerify is true (CEL-enforced companion field). Its + only purpose is to require a deliberate, separate, reviewable line in + the manifest diff before disabling TLS verification. + type: boolean + secretRef: + description: |- + SecretRef names a Secret (in the EtcdMirror's namespace) holding this + side's TLS material in the standard kubernetes.io/tls-compatible shape: + - ca.crt: PEM CA bundle used to verify the peer's server certificate + (unless CABundleRef overrides it, or InsecureSkipVerify is true). + - tls.crt / tls.key: PEM client certificate + key, for mTLS. + Optional — omit both for server-auth-only TLS. + Nil means no client identity and verification against the system trust + roots (the etcdctl default). Note a source running with + --client-cert-auth (the RKE1 default) rejects certless clients at the + handshake regardless of etcd RBAC auth; server-auth-only + Auth is not + viable against such a source. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + serverName: + description: |- + ServerName overrides the TLS ServerName (SNI) used for verification, + for cases where the dialed address doesn't match a SAN on the + certificate (e.g. dialing an NLB IP directly). Applies to EVERY + endpoint in the list, so mixing endpoints with different certificates + behind one ServerName will fail verification on the mismatched ones. + type: string + type: object + x-kubernetes-validations: + - message: insecureSkipVerify requires insecureSkipVerifyAcknowledgeRisk + to also be true + rule: '!self.insecureSkipVerify || self.insecureSkipVerifyAcknowledgeRisk' + - message: secretRef.name must be non-empty when secretRef is + set + rule: '!has(self.secretRef) || (has(self.secretRef.name) && + size(self.secretRef.name) > 0)' + type: object + x-kubernetes-validations: + - message: exactly one of endpointList or serviceRef must be set + rule: (has(self.endpointList) && size(self.endpointList) > 0) != + has(self.serviceRef) + - message: 'http:// endpoints conflict with a tls block: use https:// + endpoints or remove tls' + rule: '!(has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, + e.startsWith(''http://'')))' + - message: https:// endpoints require a tls block (an empty tls block + selects server-auth TLS against system trust roots) + rule: '!(!has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, + e.startsWith(''https://'')))' + required: + - source + - target + type: object + x-kubernetes-validations: + - message: source.prefix is immutable + rule: '(has(self.source.prefix) ? self.source.prefix : "") == (has(oldSelf.source.prefix) + ? oldSelf.source.prefix : "")' + - message: target.prefix is immutable + rule: '(has(self.target.prefix) ? self.target.prefix : "") == (has(oldSelf.target.prefix) + ? oldSelf.target.prefix : "")' + - message: sync.destPrefix is immutable + rule: '(has(self.sync) && has(self.sync.destPrefix) ? self.sync.destPrefix + : "") == (has(oldSelf.sync) && has(oldSelf.sync.destPrefix) ? oldSelf.sync.destPrefix + : "")' + - message: sync.excludePrefixes is immutable + rule: '(has(self.sync) && has(self.sync.excludePrefixes) ? self.sync.excludePrefixes + : []) == (has(oldSelf.sync) && has(oldSelf.sync.excludePrefixes) ? + oldSelf.sync.excludePrefixes : [])' + - message: checkpoint.key is immutable + rule: '(has(self.checkpoint) && has(self.checkpoint.key) ? self.checkpoint.key + : "") == (has(oldSelf.checkpoint) && has(oldSelf.checkpoint.key) ? + oldSelf.checkpoint.key : "")' + - message: checkpoint.key must live under the effective destination prefix + (target.prefix + sync.destPrefix) + rule: '!has(self.checkpoint) || !has(self.checkpoint.key) || self.checkpoint.key + == "" || self.checkpoint.key.startsWith((has(self.target.prefix) ? + self.target.prefix : "") + (has(self.sync) && has(self.sync.destPrefix) + ? self.sync.destPrefix : ""))' + status: + description: |- + EtcdMirrorStatus defines the observed state of an EtcdMirror. Progress + fields are synced periodically (not per-op), so they are a coarse, + point-in-time mirror of the agent's authoritative checkpoint in the target + etcd, useful for kubectl/dashboards, not an audit log. + + CUTOVER GATE CONTRACT: never compute "caught up" by comparing + SourceRevision to LastAppliedRevision — they snapshot at different + instants, and SourceRevision advances on out-of-prefix writes (revisions + are cluster-global). The manual gate is: quiesce source writers, read the + source's current revision R yourself (`etcdctl endpoint status`), then + poll until LastAppliedRevision >= R. The in-CR gate is spec.mode=Drain + + the CutoverReady condition. Relax target RBAC only after the mirror is + paused or deleted. + properties: + agentPod: + description: |- + AgentPod is the name of the current agent pod (the sole pod of the + size-1 Deployment), for convenient kubectl logs/exec. + type: string + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + cutover: + description: Cutover is populated while spec.mode is Drain. + properties: + drainTargetRevision: + description: |- + DrainTargetRevision is the source revision observed when Drain was + requested — the revision the watermark must reach. + format: int64 + type: integer + drainedRevision: + description: DrainedRevision is the watermark at which the drain + completed. + format: int64 + type: integer + leasedKeyCount: + description: |- + LeasedKeyCount is LeaseBackedKeyCount frozen at drain completion, for + the runbook's purge/re-lease step. + format: int64 + type: integer + sourceKeyCount: + description: |- + SourceKeyCount and TargetKeyCount are the per-side key counts from the + verification pass (source read pinned at the drained revision; + reserved checkpoint key excluded). + format: int64 + type: integer + targetKeyCount: + format: int64 + type: integer + verifiedTime: + description: VerifiedTime is when the post-drain verification + pass succeeded. + format: date-time + type: string + type: object + forcedResyncCount: + description: |- + ForcedResyncCount counts forced resyncs (compaction outran the watch, + checkpoint invalidated by a cluster-ID mismatch, or checkpoint + corrupt/unknown-version). Monotonic, never reset. + format: int32 + type: integer + initialSyncCompletionTime: + format: date-time + type: string + initialSyncKeyCount: + description: |- + InitialSyncKeyCount is the number of keys applied so far by the + current/last genesis scan. Live-updating during InitialSync (each + status sync), so InitialSyncKeyCount/InitialSyncTotalKeyCount is a + progress fraction. + format: int64 + type: integer + initialSyncStartTime: + format: date-time + type: string + initialSyncTotalKeyCount: + description: |- + InitialSyncTotalKeyCount is the denominator: the source-side key count + under the prefix observed at scan start (first page's RangeResponse + count). + format: int64 + type: integer + lastAppliedRevision: + description: |- + LastAppliedRevision is the checkpoint watermark: the source revision + through which the target is caught up, advanced by applies AND by + watch progress notifications on idle prefixes. The fenced checkpoint + key in the target etcd is the authoritative copy; this mirrors it. + format: int64 + type: integer + lastProgressTime: + description: |- + LastProgressTime is when the watermark last advanced (apply or watch + progress notification). Distinct from LastStatusSyncTime: status can + keep syncing from a wedged loop; this field is what Available and + ReplicationLagExceeded are derived from. + format: date-time + type: string + lastReconciliationDrift: + properties: + divergentKeys: + description: |- + DivergentKeys were present on both sides with different values — + distinct from MissingKeys so "a resync dropped keys" is never + conflated with "a blind window went stale". + format: int64 + type: integer + missingKeys: + description: MissingKeys were present on the source but absent + on the target. + format: int64 + type: integer + orphanKeys: + description: OrphanKeys were present on the target with no source + counterpart. + format: int64 + type: integer + repaired: + description: Repaired is true when the pass wrote fixes rather + than only reporting. + type: boolean + type: object + lastReconciliationTime: + description: |- + LastReconciliationTime and LastReconciliationDrift record the most + recent reconciliation pass (periodic or mandatory). + format: date-time + type: string + lastStatusSyncTime: + description: |- + LastStatusSyncTime is when status was last refreshed from the agent, so + staleness of everything above is directly observable. + format: date-time + type: string + leaseBackedKeyCount: + description: |- + LeaseBackedKeyCount is the number of mirrored keys whose source copy is + lease-backed (kv.Lease != 0). Mirrored copies are NOT lease-backed — + leases are stripped (see Fidelity Caveats in docs/etcdmirror.md) — so a + nonzero count means the cutover runbook's purge/re-lease step applies. + format: int64 + type: integer + observedGeneration: + format: int64 + type: integer + phase: + description: |- + EtcdMirrorPhase is a high-level summary of an EtcdMirror's lifecycle. + Unlike BackupPhase/RestorePhase, most phases here are NOT terminal — a + healthy mirror spends its life in Syncing; there is no "Completed" state. + type: string + scanRestartCount: + description: |- + ScanRestartCount counts genesis-scan attempts aborted and restarted + from a fresh R0 (watch-buffer overflow or a mid-scan watch compaction; + see the InitialSyncCompactionRaced event). Monotonic, never reset. + format: int64 + type: integer + sourceClusterID: + description: |- + SourceClusterID and TargetClusterID are the cluster IDs both bound + into the checkpoint. Either changing across reconciles is the visible + symptom of an endpoint now pointing at a different cluster than the + checkpoint was taken against (CheckpointInvalidated event, forced + genesis, RequireEmpty re-armed). + type: string + sourceKeyCount: + description: |- + SourceKeyCount and TargetKeyCount are the per-side key counts from the + most recent diff/verification pass (reserved checkpoint key and + excluded prefixes not counted; drain-verification source reads pinned + at the drained revision with a compacted-fallback re-read). Populated + by every pass that runs regardless of spec.reconciliation.enabled — + the mandatory mark-and-sweep after any forced resync, the + OverwriteAndPrune genesis pass, and the drain verification — plus the + periodic pass when it is enabled. NOT refreshed on every status sync: + a healthy RequireEmpty mirror that never forces a resync only gets + counts from an enabled periodic pass. This is the equality signal + InvariantsHeld reads; status.cutover's copies remain the frozen + drain-time snapshot. + format: int64 + type: integer + sourceRevision: + description: |- + SourceRevision is the source cluster's revision as of the last status + sync. Cluster-global: it advances on writes outside the mirrored + prefix, so SourceRevision - LastAppliedRevision OVERSTATES lag for + prefix-scoped mirrors. See the cutover gate contract above. + format: int64 + type: integer + sourceVersion: + description: |- + SourceVersion and TargetVersion are the etcd server versions from the + maintenance Status() probe at connect. + type: string + targetClusterID: + type: string + targetKeyCount: + format: int64 + type: integer + targetVersion: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index ca614b9f..11fc4953 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,6 +3,7 @@ # It should be run by config/default resources: - bases/operator.etcd.io_etcdclusters.yaml +- bases/operator.etcd.io_etcdmirrors.yaml # +kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/config/prometheus/etcdmirror_rules.yaml b/config/prometheus/etcdmirror_rules.yaml new file mode 100644 index 00000000..40e8e3c7 --- /dev/null +++ b/config/prometheus/etcdmirror_rules.yaml @@ -0,0 +1,82 @@ +# EtcdMirror alerts on the CONTROLLER-side gauges (etcd_mirror_phase, +# etcd_mirror_condition, served by the manager /metrics endpoint), so a mirror +# whose agent pod never scheduled or stopped answering still pages. +# Fuller paging algebra: docs/etcdmirror.md "Monitoring / paging algebra" and +# the condition-type comments in api/v1alpha1/etcdmirror_types.go. The +# "unless Compacted=True and progress advancing" refinement is deliberately +# omitted: it needs the agent-side +# etcd_mirror_agent_seconds_since_last_progress (agent scrape), and a +# suppressed wedged resync must still page. +# +# The manager scrape MUST set honor_labels: true (the shipped monitor.yaml +# does): the gauges carry metric-level namespace/name identifying the CR, and +# the prometheus-operator default rewrites namespace to exported_namespace — +# breaking the `on (namespace, name)` join below and the alert annotations. +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdmirror-alerts + namespace: system +spec: + groups: + - name: etcdmirror.rules + rules: + - alert: EtcdMirrorUnavailable + expr: | + etcd_mirror_condition{type="Available",status="true"} == 0 + unless on (namespace, name) etcd_mirror_phase{phase="Paused"} == 1 + for: 10m + labels: + severity: critical + annotations: + summary: 'EtcdMirror {{ $labels.namespace }}/{{ $labels.name }} not Available for 10m' + description: >- + Available has not been True for 10m and the mirror is not Paused. + status="true" == 0 covers both False and Unknown (the controller + sets Unknown when the agent /statusz poll fails). + runbook_url: https://github.com/etcd-io/etcd-operator/blob/main/docs/etcdmirror.md#monitoring--paging-algebra + - alert: EtcdMirrorTargetQuotaExhausted + expr: etcd_mirror_condition{type="TargetQuotaExhausted",status="true"} == 1 + labels: + severity: critical + annotations: + summary: 'EtcdMirror {{ $labels.namespace }}/{{ $labels.name }}: target etcd quota exhausted (NOSPACE)' + description: >- + A target write failed with NOSPACE; the agent stopped writing. + Does not self-heal: compact/defrag/disarm the target. + runbook_url: https://github.com/etcd-io/etcd-operator/blob/main/docs/etcdmirror.md#operations-error-taxonomy-and-runbook + - alert: EtcdMirrorResyncLoopDetected + expr: etcd_mirror_condition{type="ResyncLoopDetected",status="true"} == 1 + labels: + severity: critical + annotations: + summary: 'EtcdMirror {{ $labels.namespace }}/{{ $labels.name }}: forced-resync livelock' + description: >- + Consecutive forced resyncs never reached steady state — source + compaction retention < scan+drain time. Does not self-heal: + raise retention, raise sync.maxOpsPerSecond, or shrink the prefix. + runbook_url: https://github.com/etcd-io/etcd-operator/blob/main/docs/etcdmirror.md#operations-error-taxonomy-and-runbook + - alert: EtcdMirrorCertificateExpiringSoon + # Agent-side metric: INERT unless you scrape the agent pod's http + # port (8080) yourself — no agent scrape object ships with the + # operator. 14d matches the controller's certExpiryLeadWindow (the + # controller also emits CertificateExpiringSoon Warning events). + expr: (etcd_mirror_agent_tls_cert_expiry_timestamp_seconds - time()) < (14 * 86400) + for: 15m + labels: + severity: warning + annotations: + summary: >- + EtcdMirror agent cert {{ $labels.side }}/{{ $labels.kind }} + ({{ $labels.namespace }}/{{ $labels.pod }}) expires in under 14d + description: >- + A certificate referenced by the mirror expires within 14 days — + the renewal machinery (cert-manager renews ~30d out) has already + failed. The metric carries only side/kind labels; the mirror is + identified by the scrape target's namespace/pod labels. Requires + a user-provided agent scrape — inert otherwise. + runbook_url: https://github.com/etcd-io/etcd-operator/blob/main/docs/etcdmirror.md#prerequisites-summary diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml index ed137168..82bb3b7a 100644 --- a/config/prometheus/kustomization.yaml +++ b/config/prometheus/kustomization.yaml @@ -1,2 +1,3 @@ resources: - monitor.yaml +- etcdmirror_rules.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index b6cad92f..15c9292b 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -13,6 +13,11 @@ spec: - path: /metrics port: https # Ensure this is the name of the port that exposes HTTPS metrics scheme: https + # etcd_mirror_* gauges carry a metric-level `namespace` (the CR's). + # Without honorLabels the target label overwrites it (renamed to + # exported_namespace) and the rules' `on (namespace, name)` joins + # collapse to name-only — cross-namespace alert suppression. + honorLabels: true bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables diff --git a/config/rbac/etcdmirror_editor_role.yaml b/config/rbac/etcdmirror_editor_role.yaml new file mode 100644 index 00000000..a3efb459 --- /dev/null +++ b/config/rbac/etcdmirror_editor_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to edit etcdmirrors. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdmirror-editor-role +rules: +- apiGroups: + - operator.etcd.io + resources: + - etcdmirrors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - operator.etcd.io + resources: + - etcdmirrors/status + verbs: + - get diff --git a/config/rbac/etcdmirror_viewer_role.yaml b/config/rbac/etcdmirror_viewer_role.yaml new file mode 100644 index 00000000..2c62d5bd --- /dev/null +++ b/config/rbac/etcdmirror_viewer_role.yaml @@ -0,0 +1,23 @@ +# permissions for end users to view etcdmirrors. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdmirror-viewer-role +rules: +- apiGroups: + - operator.etcd.io + resources: + - etcdmirrors + verbs: + - get + - list + - watch +- apiGroups: + - operator.etcd.io + resources: + - etcdmirrors/status + verbs: + - get diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index ae91bf79..dc3d3916 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -24,4 +24,6 @@ resources: # if you do not want those helpers be installed with your Project. - etcdcluster_editor_role.yaml - etcdcluster_viewer_role.yaml +- etcdmirror_editor_role.yaml +- etcdmirror_viewer_role.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6c9ac65e..72231d2e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -28,9 +28,18 @@ rules: - list - patch - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch - apiGroups: - apps resources: + - deployments - statefulsets verbs: - create @@ -61,10 +70,18 @@ rules: - get - list - watch +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create + - patch - apiGroups: - operator.etcd.io resources: - etcdclusters + - etcdmirrors verbs: - create - delete @@ -77,12 +94,14 @@ rules: - operator.etcd.io resources: - etcdclusters/finalizers + - etcdmirrors/finalizers verbs: - update - apiGroups: - operator.etcd.io resources: - etcdclusters/status + - etcdmirrors/status verbs: - get - patch diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index f3c97ab2..23e8fdc2 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -1,4 +1,5 @@ ## Append samples of your project ## resources: - operator_v1alpha1_etcdcluster.yaml +- operator_v1alpha1_etcdmirror.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/operator_v1alpha1_etcdmirror.yaml b/config/samples/operator_v1alpha1_etcdmirror.yaml new file mode 100644 index 00000000..3a3558ec --- /dev/null +++ b/config/samples/operator_v1alpha1_etcdmirror.yaml @@ -0,0 +1,88 @@ +apiVersion: operator.etcd.io/v1alpha1 +kind: EtcdMirror +metadata: + labels: + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdmirror-sample +spec: + # Sync = continuous replication; flip to Drain at cutover time and wait for + # the CutoverReady condition. + mode: Sync + + # External RKE1/AWS source reached over a public NLB, over TLS with a + # CA-verified server certificate. https:// endpoints require a tls block + # (an empty tls block would mean system trust roots). + source: + endpointList: + - "https://etcd-rke1.example.com:2379" + prefix: "/registry/" + tls: + secretRef: + name: etcdmirror-sample-source-tls + auth: + secretRef: + name: etcdmirror-sample-source-auth + + # In-cluster EtcdCluster reached via its client Service. + target: + serviceRef: + name: etcdcluster-sample-client + port: client + prefix: "/mirrored/" + tls: + secretRef: + name: etcdmirror-sample-target-tls + + initialSync: + # RequireEmpty (default): refuse a non-empty destination prefix at + # genesis. Overwrite / OverwriteAndPrune for re-pointing at a + # previously-populated prefix (e.g. failback). + mode: RequireEmpty + + sync: + # Key rewrite — ONE formula: + # key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix) + # + # source key destPrefix resulting target key + # /registry/pods/x "" /mirrored/pods/x + # /registry/pods/x "registry/" /mirrored/registry/pods/x + # /registry/pods/x "backup/" /mirrored/backup/pods/x + # + # Default "" strips the source prefix. Immutable after creation. Leave + # unset for the default — an explicit destPrefix: "" is equivalent but + # pointless. + # destPrefix: "registry/" + # High-churn, lease-backed, or low-value ranges to skip entirely. + # Immutable after creation (it defines the mirrored range): change via + # delete-and-recreate with an appropriate initialSync.mode. + excludePrefixes: + - "/registry/events/" + maxTxnOps: 128 # includes one reserved slot for the checkpoint write + txnFlushBytes: 1Mi + pageKeyLimit: 512 + pageBytes: 1Mi + # Replay-buffer bound during the genesis scan; raise for high-churn + # sources (overflow restarts the scan from a fresh revision, surfaced + # as the InitialSyncCompactionRaced event). + # watchBufferBytes: 16Mi + maxOpsPerSecond: 500 # keep source retention > scan+drain time at this rate + requestTimeout: 30s + dialTimeout: 10s + reconnectBackoff: + initialDelay: 1s + maxDelay: 30s + + # The checkpoint lives in the TARGET etcd, fenced and written with each + # batch. Default key: + "\x00etcdmirror-checkpoint". + # An override MUST stay under the effective destination prefix (the + # range-scoped target credential covers it; CEL-enforced) — pick a + # different suffix under it if the default collides with your own + # reserved-key scheme. + # checkpoint: + # key: "/mirrored/\x00my-own-checkpoint-name" + + reconciliation: + enabled: true + interval: 1h + deleteOrphans: false diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index 6a32be4e..7c2e26f0 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -11,6 +11,8 @@ Package v1alpha1 contains API Schema definitions for the operator v1alpha1 API g ### Resource Types - [EtcdCluster](#etcdcluster) - [EtcdClusterList](#etcdclusterlist) +- [EtcdMirror](#etcdmirror) +- [EtcdMirrorList](#etcdmirrorlist) @@ -115,6 +117,444 @@ _Appears in:_ +#### EtcdMirror + + + +EtcdMirror is the Schema for the etcdmirrors API. It describes a +continuous, one-way key-range sync from a source etcd cluster to a target +etcd cluster, run as a single supervised stateless pod (a size-1 +Deployment; progress lives in a fenced checkpoint key in the target etcd, +not on a volume). + +It is a byte-copy of keys and values, not a replica: revisions, versions, +and create/mod ordering are target-assigned, and leases are stripped. See +Fidelity Caveats in docs/etcdmirror.md before depending on anything but +key/value content. + +Two-way sync: never — etcd revisions are cluster-local and there is no +per-key provenance channel, so bidirectional sync is structurally +inexpressible. Reversal for cutover/failback: yes — delete the CR and +create a new one with swapped endpoints, initialSync.mode +OverwriteAndPrune, only after the forward mirror reported CutoverReady. +See docs/etcdmirror.md. + + + +_Appears in:_ +- [EtcdMirrorList](#etcdmirrorlist) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `operator.etcd.io/v1alpha1` | | | +| `kind` _string_ | `EtcdMirror` | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[EtcdMirrorSpec](#etcdmirrorspec)_ | | | | + + +#### EtcdMirrorAuth + + + +EtcdMirrorAuth configures etcd RBAC username/password auth for one side. + + + +_Appears in:_ +- [EtcdMirrorEndpoint](#etcdmirrorendpoint) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `secretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#localobjectreference-v1-core)_ | SecretRef names a Secret holding "username" and "password" keys. If
this whole Auth block is nil, the agent does not call etcd's
Authenticate() at all; the pinned v3 client transparently re-auths on
token expiry. PRECEDENCE: when both a client certificate and Auth are
supplied, etcd uses the token identity, not the certificate CN — the
Auth user must hold the range-scoped role. | | | + + +#### EtcdMirrorBackoffSpec + + + + + + + +_Appears in:_ +- [EtcdMirrorSyncSpec](#etcdmirrorsyncspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `initialDelay` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | | | Optional: \{\}
| +| `maxDelay` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | | | Optional: \{\}
| + + +#### EtcdMirrorCABundleRef + + + +EtcdMirrorCABundleRef points at one key of a Secret or ConfigMap holding a +PEM CA bundle. + + + +_Appears in:_ +- [EtcdMirrorTLS](#etcdmirrortls) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `kind` _string_ | Kind is Secret or ConfigMap. Defaults to ConfigMap. | ConfigMap | Enum: [Secret ConfigMap]
Optional: \{\}
| +| `name` _string_ | Name of the Secret or ConfigMap, in the EtcdMirror's namespace. | | MinLength: 1
| +| `key` _string_ | Key within the object. Defaults to "ca.crt". | | Optional: \{\}
| + + +#### EtcdMirrorCheckpointSpec + + + +EtcdMirrorCheckpointSpec configures the reserved checkpoint/fence key the +agent maintains IN THE TARGET etcd. The checkpoint (the source-revision +watermark plus {linkUID, epoch, role}) is written in the SAME Txn as every +applied batch and fenced with a mod-revision compare on EVERY write path +(applies, reconciliation repairs, prune deletes), so two agents can never +interleave writes and a straggler apply after cutover fails loudly. The +key is excluded by exact match from scans, counts, prune passes, and the +RequireEmpty check; the target RBAC grant must cover it; CR deletion +removes it via a delete-one-key finalizer. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `key` _string_ | Key overrides the reserved checkpoint key. Defaults to the effective
destination prefix + "\x00etcdmirror-checkpoint" — the \x00 byte after
the prefix cannot collide with any real key under it. MUST live under
the effective destination prefix (target.prefix + sync.destPrefix,
CEL-enforced): the range-scoped target credential covers it, and the
exact-match exclusion from scans/counts/prune only works inside the
mirrored range. Immutable after creation. | | Optional: \{\}
| + + +#### EtcdMirrorCutoverStatus + + + +EtcdMirrorCutoverStatus tracks a Drain-mode cutover. + + + +_Appears in:_ +- [EtcdMirrorStatus](#etcdmirrorstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `drainTargetRevision` _integer_ | DrainTargetRevision is the source revision observed when Drain was
requested — the revision the watermark must reach. | | Optional: \{\}
| +| `drainedRevision` _integer_ | DrainedRevision is the watermark at which the drain completed. | | Optional: \{\}
| +| `verifiedTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#time-v1-meta)_ | VerifiedTime is when the post-drain verification pass succeeded. | | Optional: \{\}
| +| `sourceKeyCount` _integer_ | SourceKeyCount and TargetKeyCount are the per-side key counts from the
verification pass (source read pinned at the drained revision;
reserved checkpoint key excluded). | | Optional: \{\}
| +| `targetKeyCount` _integer_ | | | Optional: \{\}
| +| `leasedKeyCount` _integer_ | LeasedKeyCount is LeaseBackedKeyCount frozen at drain completion, for
the runbook's purge/re-lease step. | | Optional: \{\}
| + + +#### EtcdMirrorDriftInfo + + + + + + + +_Appears in:_ +- [EtcdMirrorStatus](#etcdmirrorstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `missingKeys` _integer_ | MissingKeys were present on the source but absent on the target. | | | +| `divergentKeys` _integer_ | DivergentKeys were present on both sides with different values —
distinct from MissingKeys so "a resync dropped keys" is never
conflated with "a blind window went stale". | | | +| `orphanKeys` _integer_ | OrphanKeys were present on the target with no source counterpart. | | | +| `repaired` _boolean_ | Repaired is true when the pass wrote fixes rather than only reporting. | | | + + +#### EtcdMirrorEndpoint + + + +EtcdMirrorEndpoint describes how to reach, authenticate to, and scope one +side (source or target) of a mirror. Both sides need an identical shape +(address resolution + prefix + TLS + auth), so one type serves both roles, +the same way BackupDestination is reused verbatim between EtcdBackup and +EtcdRestore rather than forked into near-duplicate per-role types. + +Exactly one of EndpointList or ServiceRef must be set. An empty +endpointList ([]) is treated as unset, per Kubernetes list conventions — +so `endpointList: []` alongside a serviceRef is accepted. + +Endpoint scheme and the TLS block must agree (CEL-enforced both ways): +http:// endpoints with a tls block would silently drop TLS at dial time; +https:// endpoints without one would dial with undeclared system-roots +TLS. The agent derives the dial scheme from the presence of the tls block, +so the declared contract is true by construction. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `endpointList` _string array_ | EndpointList is a raw set of etcd client-URL host:port (or
scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379".
This is the ONLY supported mechanism for a cluster external to this
Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB);
there is deliberately no tunnel/port-forward mode — terminate any
tunnel upstream and hand EtcdMirror the resulting stable endpoint(s).
Prefer listing per-member endpoints over a single load-balancer VIP: a
TCP-health-checked VIP cannot see etcd quorum, and the client's own
balancer handles per-member failover.
IP-LITERAL ENDPOINTS: Go's TLS stack requires an IP SAN (not a DNS SAN)
to verify a bare-IP endpoint. Either set EtcdMirrorTLS.ServerName to a
hostname present as a DNS SAN on the certificate, or ensure the
certificate carries a matching IP SAN. Fix the SAN/ServerName mismatch;
do not reach for InsecureSkipVerify. | | Optional: \{\}
| +| `serviceRef` _[EtcdMirrorServiceRef](#etcdmirrorserviceref)_ | ServiceRef points at a Kubernetes Service in this cluster whose DNS
name resolves the etcd client endpoint(s), for the same-cluster case.
Namespace defaults to the EtcdMirror's own namespace when empty. | | Optional: \{\}
| +| `prefix` _string_ | Prefix is the etcd key prefix on THIS side. On Source, only keys under
this prefix are synced; empty means the whole keyspace. On Target, this
is the prefix under which mirrored keys land (see EtcdMirrorSyncSpec's
rewrite formula). Immutable after creation. | | Optional: \{\}
| +| `tls` _[EtcdMirrorTLS](#etcdmirrortls)_ | TLS configures the agent's client TLS for THIS side. Nil means the
agent dials this side in cleartext (and https:// endpoints are
CEL-rejected). An empty block means server-auth TLS verified against
the system trust roots. | | Optional: \{\}
| +| `auth` _[EtcdMirrorAuth](#etcdmirrorauth)_ | Auth configures etcd username/password (RBAC) auth for THIS side. | | Optional: \{\}
| + + +#### EtcdMirrorInitialSyncMode + +_Underlying type:_ _string_ + +EtcdMirrorInitialSyncMode governs how the agent treats pre-existing keys +under the effective destination prefix at genesis (first-ever sync, or a +checkpoint invalidated by a cluster-identity mismatch). + + + +_Appears in:_ +- [EtcdMirrorInitialSyncSpec](#etcdmirrorinitialsyncspec) + +| Field | Description | +| --- | --- | +| `RequireEmpty` | EtcdMirrorInitialSyncRequireEmpty refuses to start if the destination
prefix already holds any key (Phase -> Failed, condition
EmptyTargetViolation). The reserved checkpoint key is excluded by exact
match.
RE-ARM CONTRACT: a source OR target cluster-ID mismatch invalidates
the checkpoint, forces genesis, and RE-ARMS this check. An ordinary
forced resync (Compacted) does NOT re-check RequireEmpty: the decoded,
ownership-validated fence proves the destination data is this link's
own.
| +| `Overwrite` | EtcdMirrorInitialSyncOverwrite scans and writes over whatever is there.
Keys present on the target but absent on the source are left alone.
| +| `OverwriteAndPrune` | EtcdMirrorInitialSyncOverwriteAndPrune is Overwrite plus one mandatory
orphan-prune pass after the scan: target keys under the destination
prefix with no source counterpart are deleted. This makes reversal onto
a previously-populated prefix (failback) a first-class correct
operation instead of silently resurrecting deleted keys.
| + + +#### EtcdMirrorInitialSyncSpec + + + +EtcdMirrorInitialSyncSpec governs the genesis scan. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `mode` _[EtcdMirrorInitialSyncMode](#etcdmirrorinitialsyncmode)_ | Mode governs pre-existing destination keys at genesis. Defaults to
RequireEmpty (refuse a non-empty destination prefix). | RequireEmpty | Enum: [RequireEmpty Overwrite OverwriteAndPrune]
Optional: \{\}
| +| `startRevision` _integer_ | StartRevision, when > 0, skips the genesis scan entirely and starts
watching from StartRevision+1. For fidelity-preserving seeds: restore
the target from a source snapshot (`etcdutl snapshot restore
--bump-revision --mark-compacted`), then mirror only the delta.
Requires Mode Overwrite or OverwriteAndPrune (CEL-enforced). | | Minimum: 0
Optional: \{\}
| + + +#### EtcdMirrorList + + + +EtcdMirrorList contains a list of EtcdMirror. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `operator.etcd.io/v1alpha1` | | | +| `kind` _string_ | `EtcdMirrorList` | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[EtcdMirror](#etcdmirror) array_ | | | | + + +#### EtcdMirrorMode + +_Underlying type:_ _string_ + +EtcdMirrorMode selects the mirror's operating mode. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | +| --- | --- | +| `Sync` | EtcdMirrorModeSync is normal continuous replication.
| +| `Drain` | EtcdMirrorModeDrain prepares for cutover: the agent records the source
revision observed when Drain is requested (status.cutover.drainTargetRevision),
keeps replicating until the checkpoint watermark reaches it, runs a
verification pass (per-side key counts, lease-backed key count), then
sets the CutoverReady condition and flips the fence key's role to
Primary so any straggler apply fails its mod-revision compare loudly.
Runbook: quiesce source writers -> set mode=Drain ->
`kubectl wait --for=condition=CutoverReady etcdmirror/` ->
purge/re-lease lease-backed keys -> repoint clients -> delete the CR.
| + + +#### EtcdMirrorPhase + +_Underlying type:_ _string_ + +EtcdMirrorPhase is a high-level summary of an EtcdMirror's lifecycle. +Unlike BackupPhase/RestorePhase, most phases here are NOT terminal — a +healthy mirror spends its life in Syncing; there is no "Completed" state. + + + +_Appears in:_ +- [EtcdMirrorStatus](#etcdmirrorstatus) + +| Field | Description | +| --- | --- | +| `Pending` | EtcdMirrorPhasePending means the EtcdMirror has been accepted but the
agent workload has not been created yet.
| +| `Connecting` | EtcdMirrorPhaseConnecting means the agent pod is running, establishing
client connections to both sides and probing versions/cluster IDs.
| +| `InitialSync` | EtcdMirrorPhaseInitialSync means the agent is running the genesis scan:
an UNPINNED chunked scan with the watch already open from the revision
observed before the scan started, buffered events replayed over the
scanned base (reflector pattern). Because pages read at the current
revision, mid-scan compaction cannot fail the scan. Also entered during
a forced resync (then with condition Compacted=True/Reason=ForcedResync).
| +| `Syncing` | EtcdMirrorPhaseSyncing is the steady state: watching and applying live
changes, watermark advancing via progress notifications. A completed
drain also reports Syncing (there is no Drained phase — a finished
cutover is not page-worthy); the CutoverReady condition carries the
drain outcome.
| +| `Degraded` | EtcdMirrorPhaseDegraded means the agent is in a retry/backoff loop
(connection or throttling class) and is expected to self-heal. Forced
resyncs are NOT Degraded; they report as InitialSync + Compacted=True.
| +| `Paused` | EtcdMirrorPhasePaused means spec.paused is true; the agent Deployment
is scaled to zero. The checkpoint is retained in the target.
| +| `Failed` | EtcdMirrorPhaseFailed means a terminal, non-recoverable error requiring
operator intervention: EmptyTargetViolation at genesis, source below
the 3.4 version floor (UnsupportedVersion), a permanent-class write
error (oversized revision vs target limits), malformed cert material,
or unresolvable spec misconfiguration.
| + + +#### EtcdMirrorReconciliationSpec + + + +EtcdMirrorReconciliationSpec configures the periodic full reconciliation +pass. The same engine also runs unconditionally (regardless of Enabled or +DeleteOrphans) as the post-forced-resync mark-and-sweep, the +OverwriteAndPrune genesis pass, and the Drain verification pass. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `enabled` _boolean_ | Enabled toggles the PERIODIC pass. Defaults to false: it is a full
diff of the prefix contents on both sides (O(keyspace)), so it is
opt-in. | | Optional: \{\}
| +| `interval` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | Interval between periodic passes. Defaults to 1h when Enabled. | | Optional: \{\}
| +| `deleteOrphans` _boolean_ | DeleteOrphans, when true, allows the PERIODIC pass to delete target
keys under the destination prefix that have no corresponding source
key. Defaults to false. (Forced-resync sweeps and OverwriteAndPrune
always delete orphans; this knob only governs the periodic pass.) | | Optional: \{\}
| + + +#### EtcdMirrorServiceRef + + + +EtcdMirrorServiceRef points at a Service and the client port on it to dial. + + + +_Appears in:_ +- [EtcdMirrorEndpoint](#etcdmirrorendpoint) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | Name is the Service name. | | MinLength: 1
| +| `namespace` _string_ | Namespace defaults to the EtcdMirror's own namespace when empty. | | Optional: \{\}
| +| `port` _string_ | Port is the Service port name or number exposing etcd's client API.
Defaults to "client" when empty, matching PodMonitorSpec.Port's
convention elsewhere in this API group. | | Optional: \{\}
| + + +#### EtcdMirrorSpec + + + +EtcdMirrorSpec defines the desired state of an EtcdMirror. + +Range-defining and rewrite fields are immutable (CEL transition rules +below): source.prefix, target.prefix, sync.destPrefix, sync.excludePrefixes +and checkpoint.key. Changing what range is mirrored, or where it lands, +mid-life silently diverges: a restarted agent resumes from its checkpoint +without a scan, so removing an exclusion never backfills pre-existing keys +and adding one strands already-mirrored keys as permanent orphans — all +with every condition green. Endpoints stay mutable — rotating an NLB DNS +name or adding a member to the same cluster is routine; pointing at a +different cluster is caught at runtime by the checkpoint's dual-cluster-ID +binding, not by spec validation. + +The transition rules compare VALUES, presence-normalized: for these +fields the empty value and an absent field are semantically identical +(prefix "" = whole keyspace, destPrefix "" = strip the source prefix), and +Go typed clients drop explicit "" through omitempty — presence-based rules +would reject every typed-client update of a CR created with an explicit +empty string. + + + +_Appears in:_ +- [EtcdMirror](#etcdmirror) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `mode` _[EtcdMirrorMode](#etcdmirrormode)_ | Mode selects continuous replication (Sync, the default) or a cutover
drain (Drain). See EtcdMirrorModeDrain for the cutover contract. | Sync | Enum: [Sync Drain]
Optional: \{\}
| +| `source` _[EtcdMirrorEndpoint](#etcdmirrorendpoint)_ | Source is the etcd cluster keys are read from. EtcdMirror never writes
back to Source; the agent's source-side client is only ever used for
Get/Watch, never Put/Delete/Txn.
VERSION FLOOR: source etcd must be >= 3.4 (probed via maintenance
Status() at connect; below the floor the mirror goes Failed with reason
UnsupportedVersion). >= 3.4.25 / 3.5.8 is the recommended floor: below
it, watch progress notifications are unreliable and the agent cannot
trust the watermark machinery that drives lag, the checkpoint, and the
Drain gate. | | | +| `target` _[EtcdMirrorEndpoint](#etcdmirrorendpoint)_ | Target is the etcd cluster keys are written into.
SECURITY PREREQUISITE: Target's credential (etcd RBAC user and/or the
client certificate's role) MUST be range-scoped to the effective
destination prefix — never a cluster-admin-equivalent credential. The
agent's client-side rewrite logic is defense against bugs in its own
code, NOT a security boundary. The grant must also cover the reserved
checkpoint key (see Checkpoint). Configure via `etcdctl role
grant-permission --prefix=true readwrite ` before
pointing an EtcdMirror at the cluster.
The target must run with auto-compaction enabled: forced-resync churn
and prune passes march an uncompacted target toward its storage quota
(2GiB by default), which surfaces as TargetQuotaExhausted. | | | +| `initialSync` _[EtcdMirrorInitialSyncSpec](#etcdmirrorinitialsyncspec)_ | InitialSync governs genesis behavior: how pre-existing destination keys
are treated (Mode) and optionally where replication starts
(StartRevision). | | Optional: \{\}
| +| `sync` _[EtcdMirrorSyncSpec](#etcdmirrorsyncspec)_ | Sync tunes runtime sync behavior (batching, paging, rate limiting,
prefix rewrite, timeouts, backoff). | | Optional: \{\}
| +| `checkpoint` _[EtcdMirrorCheckpointSpec](#etcdmirrorcheckpointspec)_ | Checkpoint configures the reserved checkpoint/fence key on the target. | | Optional: \{\}
| +| `reconciliation` _[EtcdMirrorReconciliationSpec](#etcdmirrorreconciliationspec)_ | Reconciliation optionally enables a periodic full diff-and-repair pass
layered on top of the continuous watch-based mirror. Independent of
this setting, one reconciliation-with-delete pass always runs after any
forced resync (mark-and-sweep), and as the OverwriteAndPrune genesis
pass and the Drain verification pass. | | Optional: \{\}
| +| `podTemplate` _[PodTemplate](#podtemplate)_ | PodTemplate carries scheduling/affinity/labels/annotations for the
agent pod, reusing EtcdClusterSpec's PodTemplate shape verbatim. | | Optional: \{\}
| +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#resourcerequirements-v1-core)_ | Resources are the agent container's compute resources. The agent's
memory model is bounded by Sync.PageBytes (single in-flight scan page,
no unbounded read-ahead); size limits accordingly. | | Optional: \{\}
| +| `paused` _boolean_ | Paused, when true, scales the agent Deployment to zero without deleting
the CR or its checkpoint. The checkpoint lives in the target etcd, so
resume picks up from the last fenced watermark. NOTE: pausing longer
than the source's compaction retention guarantees a full forced resync
on resume — there is no free lunch past the retention window. | | Optional: \{\}
| + + + + +#### EtcdMirrorSyncSpec + + + +EtcdMirrorSyncSpec tunes the mirror's runtime sync behavior. + +KEY REWRITE — one formula, no other composition: + + key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix) + +(anchored strip-and-reprefix; never a substring replace). + +BATCHING INVARIANT: target Txns flush ONLY at source-revision boundaries — +a source revision's events are never split across Txns, whole revisions +are coalesced up to the MaxTxnOps/TxnFlushBytes watermarks, and one op +slot in MaxTxnOps is always reserved for the checkpoint write that rides +in the same Txn. A single source revision larger than MaxTxnOps is applied +as one oversized Txn (provision the target's --max-txn-ops accordingly) +with the checkpoint held until it lands. + +RETENTION PREREQUISITE: the source's compaction retention window must +exceed the worst-case initial scan + throttled drain time, approximately +sourceKeyCount / min(effective scan rate, MaxOpsPerSecond). If it does +not, genesis (and every forced resync) loses the race with compaction and +the mirror livelocks (surfaced via the resync-loop detector). + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `destPrefix` _string_ | DestPrefix is the middle term of the rewrite formula above. Default ""
means the source prefix is stripped and key remainders land directly
under target.prefix. Immutable after creation. | | Optional: \{\}
| +| `excludePrefixes` _string array_ | ExcludePrefixes lists source key prefixes (full source-side keys, e.g.
"/registry/events/") skipped entirely: not scanned, not watched, not
counted, not pruned. Use to drop high-churn low-value ranges and cut
WAN cost, or to skip lease-backed ranges that don't survive mirroring.
Nested/duplicate entries are normalized by the agent (an entry covered
by another is dropped). Immutable after creation (it defines the
mirrored range): removing an exclusion would require a backfill scan
the checkpoint-resume path never runs, and adding one would strand
already-mirrored keys as permanent orphans — change it via
delete-and-recreate with an appropriate initialSync.mode instead. | | MaxItems: 64
Optional: \{\}
| +| `maxTxnOps` _integer_ | MaxTxnOps bounds how many operations the agent batches into a single
target Txn, including the reserved checkpoint-write slot. Must not
exceed the target's --max-txn-ops (etcd default 128). Defaults to 128. | | Minimum: 2
Optional: \{\}
| +| `txnFlushBytes` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#quantity-resource-api)_ | TxnFlushBytes is the byte watermark at which a batch is flushed (at the
next source-revision boundary). Keep well under etcd's request size
limits: a Txn over ~1.5MiB is rejected by the server and one over 2MiB
by the client send cap — both classified permanent errors, not
throttling. Defaults to 1Mi. | | Optional: \{\}
| +| `pageKeyLimit` _integer_ | PageKeyLimit bounds keys per source scan page during InitialSync and
reconciliation. The scan is pull-based, one page in flight — no
read-ahead — so this and PageBytes bound agent memory. Defaults to 512. | | Minimum: 1
Optional: \{\}
| +| `pageBytes` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#quantity-resource-api)_ | PageBytes bounds bytes per source scan page. Defaults to 1Mi. | | Optional: \{\}
| +| `watchBufferBytes` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#quantity-resource-api)_ | WatchBufferBytes bounds the memory used to buffer watch events
observed from R0 while the genesis scan runs (the reflector replay
buffer). On overflow the agent cancels the source watch and restarts
the scan from a fresh R0 (see the InitialSyncCompactionRaced event) —
a bounded retry instead of unbounded growth when source churn outruns
scan+apply throughput. Defaults to 16Mi (must stay in lockstep with
pkg/mirroragent's DefaultWatchBufferBytes). | | Optional: \{\}
| +| `maxOpsPerSecond` _integer_ | MaxOpsPerSecond rate-limits the agent's target write rate (a token
bucket over puts+deletes/sec), applied to both the genesis scan and
watch-driven applies. Zero (default) means unlimited. Mind the
retention prerequisite above when throttling. | | Minimum: 0
Optional: \{\}
| +| `requestTimeout` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | RequestTimeout is the per-RPC context deadline applied to every unary
call on both sides (watches excluded — they are long-lived by design
and covered by progress-notification liveness instead). Without it a
blackholed call through an NLB never errors and backoff never engages.
Defaults to 30s. | | Optional: \{\}
| +| `dialTimeout` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | DialTimeout bounds establishing the initial client connection to each
side. Defaults to 10s. | | Optional: \{\}
| +| `reconnectBackoff` _[EtcdMirrorBackoffSpec](#etcdmirrorbackoffspec)_ | ReconnectBackoff bounds the retry/backoff loop wrapping connection-class
errors. Throttling-class errors (target rate rejection) use a more
conservative curve derived from the same bounds; quota exhaustion
(TargetQuotaExhausted) and permanent errors are never retried through
this loop. Defaults to exponential backoff from 1s to 30s. | | Optional: \{\}
| + + +#### EtcdMirrorTLS + + + +EtcdMirrorTLS configures the mirror agent's client TLS for one side of a +mirror. Plain secretRef, not a reuse of EtcdClusterTLS/TLSSurface — the +agent is always a client to clusters it does not own, so issuer-selection +machinery doesn't apply. + +ROTATION CONTRACT: the agent re-reads TLS material from the mounted Secret +on every handshake (transport.TLSInfo file paths, not a one-shot +tls.Config); certificate rotation requires no pod restart. + + + +_Appears in:_ +- [EtcdMirrorEndpoint](#etcdmirrorendpoint) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `secretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#localobjectreference-v1-core)_ | SecretRef names a Secret (in the EtcdMirror's namespace) holding this
side's TLS material in the standard kubernetes.io/tls-compatible shape:
- ca.crt: PEM CA bundle used to verify the peer's server certificate
(unless CABundleRef overrides it, or InsecureSkipVerify is true).
- tls.crt / tls.key: PEM client certificate + key, for mTLS.
Optional — omit both for server-auth-only TLS.
Nil means no client identity and verification against the system trust
roots (the etcdctl default). Note a source running with
--client-cert-auth (the RKE1 default) rejects certless clients at the
handshake regardless of etcd RBAC auth; server-auth-only + Auth is not
viable against such a source. | | Optional: \{\}
| +| `caBundleRef` _[EtcdMirrorCABundleRef](#etcdmirrorcabundleref)_ | CABundleRef optionally sources the trust anchors from a separate
Secret or ConfigMap key, decoupling trust from the identity Secret
(Gateway API caCertificateRefs precedent). Takes precedence over
SecretRef's ca.crt. | | Optional: \{\}
| +| `insecureSkipVerify` _boolean_ | InsecureSkipVerify disables server certificate verification. Strongly
discouraged, especially for a source reached over the public internet.
Requires InsecureSkipVerifyAcknowledgeRisk to also be set true
(CEL-enforced). The controller additionally emits a standing Warning
event whenever this is true. | false | Optional: \{\}
| +| `insecureSkipVerifyAcknowledgeRisk` _boolean_ | InsecureSkipVerifyAcknowledgeRisk must independently be set true
whenever InsecureSkipVerify is true (CEL-enforced companion field). Its
only purpose is to require a deliberate, separate, reviewable line in
the manifest diff before disabling TLS verification. | false | Optional: \{\}
| +| `serverName` _string_ | ServerName overrides the TLS ServerName (SNI) used for verification,
for cases where the dialed address doesn't match a SAN on the
certificate (e.g. dialing an NLB IP directly). Applies to EVERY
endpoint in the list, so mixing endpoints with different certificates
behind one ServerName will fail verification on the mismatched ones. | | Optional: \{\}
| + + #### MemberStatus @@ -153,6 +593,24 @@ _Appears in:_ | `labels` _object (keys:string, values:string)_ | | | | +#### PodSpec + + + + + + + +_Appears in:_ +- [PodTemplate](#podtemplate) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#affinity-v1-core)_ | | | | +| `nodeSelector` _object (keys:string, values:string)_ | | | | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#toleration-v1-core) array_ | | | | + + #### PodTemplate @@ -163,10 +621,12 @@ _Appears in:_ _Appears in:_ - [EtcdClusterSpec](#etcdclusterspec) +- [EtcdMirrorSpec](#etcdmirrorspec) | Field | Description | Default | Validation | | --- | --- | --- | --- | | `metadata` _[PodMetadata](#podmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[PodSpec](#podspec)_ | | | | #### ProviderAutoConfig diff --git a/docs/etcdmirror-dashboard.json b/docs/etcdmirror-dashboard.json new file mode 100644 index 00000000..c607362f --- /dev/null +++ b/docs/etcdmirror-dashboard.json @@ -0,0 +1,249 @@ +{ + "title": "EtcdMirror", + "uid": "etcd-mirror", + "tags": ["etcd", "etcd-operator", "etcdmirror"], + "schemaVersion": 39, + "editable": true, + "graphTooltip": 1, + "time": {"from": "now-6h", "to": "now"}, + "refresh": "30s", + "templating": { + "list": [ + { + "name": "datasource", + "label": "Data source", + "type": "datasource", + "query": "prometheus" + }, + { + "name": "namespace", + "type": "query", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "query": "label_values(etcd_mirror_phase, namespace)", + "refresh": 2, + "sort": 1 + }, + { + "name": "name", + "type": "query", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "query": "label_values(etcd_mirror_phase{namespace=\"$namespace\"}, name)", + "refresh": 2, + "sort": 1 + } + ] + }, + "panels": [ + { + "id": 1, + "type": "text", + "title": "", + "gridPos": {"h": 3, "w": 24, "x": 0, "y": 0}, + "options": { + "mode": "markdown", + "content": "**Phase** and **Conditions** are controller-side (manager `/metrics`): present even when the agent pod never scheduled. All other panels use `etcd_mirror_agent_*` metrics and require the agent pods' `http` port (8080) to be scraped with `namespace`/`pod` labels — no scrape object ships with the operator. Paging algebra: `docs/etcdmirror.md`." + } + }, + { + "id": 2, + "type": "state-timeline", + "title": "Phase (controller-side)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 7, "w": 12, "x": 0, "y": 3}, + "targets": [ + { + "refId": "A", + "expr": "etcd_mirror_phase{namespace=\"$namespace\",name=\"$name\"} == 1", + "legendFormat": "{{phase}}" + } + ], + "fieldConfig": { + "defaults": {"custom": {"fillOpacity": 70, "lineWidth": 0}, "mappings": []}, + "overrides": [] + }, + "options": {"showValue": "never", "legend": {"displayMode": "list", "placement": "bottom"}} + }, + { + "id": 3, + "type": "table", + "title": "Conditions (controller-side)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 7, "w": 12, "x": 12, "y": 3}, + "targets": [ + { + "refId": "A", + "expr": "etcd_mirror_condition{namespace=\"$namespace\",name=\"$name\"} == 1", + "instant": true, + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {"Time": true, "Value": true, "__name__": true, "instance": true, "job": true, "namespace": true, "name": true, "pod": true, "service": true, "endpoint": true, "container": true}, + "indexByName": {"type": 0, "status": 1} + } + } + ], + "fieldConfig": {"defaults": {"mappings": []}, "overrides": []}, + "options": {"footer": {"show": false}} + }, + { + "id": 4, + "type": "timeseries", + "title": "Replication lag (revisions)", + "description": "max(0, source_revision - watermark). Overstates lag for prefix-scoped mirrors: revisions are cluster-global.", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 7, "w": 8, "x": 0, "y": 10}, + "targets": [ + { + "refId": "A", + "expr": "etcd_mirror_agent_lag_revisions{namespace=\"$namespace\",pod=~\"$name-mirror-agent-.*\"}", + "legendFormat": "{{pod}}" + } + ], + "fieldConfig": {"defaults": {"unit": "short", "min": 0}, "overrides": []} + }, + { + "id": 5, + "type": "timeseries", + "title": "Seconds since last progress", + "description": "Watermark staleness. The controller marks Available=False (ProgressStalled) past 3x the progress-notification interval.", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 7, "w": 8, "x": 8, "y": 10}, + "targets": [ + { + "refId": "A", + "expr": "etcd_mirror_agent_seconds_since_last_progress{namespace=\"$namespace\",pod=~\"$name-mirror-agent-.*\"}", + "legendFormat": "{{pod}}" + } + ], + "fieldConfig": {"defaults": {"unit": "s", "min": 0}, "overrides": []} + }, + { + "id": 6, + "type": "timeseries", + "title": "Apply rate (ops/s)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 7, "w": 8, "x": 16, "y": 10}, + "targets": [ + { + "refId": "A", + "expr": "rate(etcd_mirror_agent_keys_applied_total{namespace=\"$namespace\",pod=~\"$name-mirror-agent-.*\"}[5m])", + "legendFormat": "{{pod}}" + } + ], + "fieldConfig": {"defaults": {"unit": "ops", "min": 0}, "overrides": []} + }, + { + "id": 7, + "type": "stat", + "title": "Initial sync progress", + "description": "initial_sync_keys / initial_sync_expected_keys. Only meaningful during a genesis scan or forced resync.", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 7, "w": 6, "x": 0, "y": 17}, + "targets": [ + { + "refId": "A", + "expr": "etcd_mirror_agent_initial_sync_keys{namespace=\"$namespace\",pod=~\"$name-mirror-agent-.*\"} / etcd_mirror_agent_initial_sync_expected_keys{namespace=\"$namespace\",pod=~\"$name-mirror-agent-.*\"}" + } + ], + "fieldConfig": { + "defaults": {"unit": "percentunit", "min": 0, "max": 1, "mappings": []}, + "overrides": [] + }, + "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}} + }, + { + "id": 8, + "type": "timeseries", + "title": "Forced resyncs by reason (1h)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 7, "w": 9, "x": 6, "y": 17}, + "targets": [ + { + "refId": "A", + "expr": "increase(etcd_mirror_agent_forced_resync_total{namespace=\"$namespace\",pod=~\"$name-mirror-agent-.*\"}[1h])", + "legendFormat": "{{reason}}" + } + ], + "fieldConfig": {"defaults": {"unit": "short", "min": 0}, "overrides": []} + }, + { + "id": 9, + "type": "timeseries", + "title": "Drift (last full reconciliation diff)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 7, "w": 9, "x": 15, "y": 17}, + "targets": [ + { + "refId": "A", + "expr": "etcd_mirror_agent_drift_missing_keys{namespace=\"$namespace\",pod=~\"$name-mirror-agent-.*\"}", + "legendFormat": "missing" + }, + { + "refId": "B", + "expr": "etcd_mirror_agent_drift_divergent_keys{namespace=\"$namespace\",pod=~\"$name-mirror-agent-.*\"}", + "legendFormat": "divergent" + }, + { + "refId": "C", + "expr": "etcd_mirror_agent_drift_orphan_keys{namespace=\"$namespace\",pod=~\"$name-mirror-agent-.*\"}", + "legendFormat": "orphan" + } + ], + "fieldConfig": {"defaults": {"unit": "short", "min": 0}, "overrides": []} + }, + { + "id": 10, + "type": "timeseries", + "title": "Source vs target key counts", + "description": "In-scope counts from the most recent reconciliation/verification pass (reserved checkpoint key excluded).", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 7, "w": 12, "x": 0, "y": 24}, + "targets": [ + { + "refId": "A", + "expr": "etcd_mirror_agent_source_keys{namespace=\"$namespace\",pod=~\"$name-mirror-agent-.*\"}", + "legendFormat": "source" + }, + { + "refId": "B", + "expr": "etcd_mirror_agent_target_keys{namespace=\"$namespace\",pod=~\"$name-mirror-agent-.*\"}", + "legendFormat": "target" + } + ], + "fieldConfig": {"defaults": {"unit": "short", "min": 0}, "overrides": []} + }, + { + "id": 11, + "type": "timeseries", + "title": "TLS cert expiry (days remaining)", + "description": "Warning alert fires under 14d — the controller's certExpiryLeadWindow.", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 7, "w": 12, "x": 12, "y": 24}, + "targets": [ + { + "refId": "A", + "expr": "(etcd_mirror_agent_tls_cert_expiry_timestamp_seconds{namespace=\"$namespace\",pod=~\"$name-mirror-agent-.*\"} - time()) / 86400", + "legendFormat": "{{side}}/{{kind}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "d", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "yellow", "value": 14}, + {"color": "green", "value": 30} + ] + } + }, + "overrides": [] + } + } + ] +} diff --git a/docs/etcdmirror.md b/docs/etcdmirror.md new file mode 100644 index 00000000..dfcb291d --- /dev/null +++ b/docs/etcdmirror.md @@ -0,0 +1,158 @@ +# EtcdMirror + +`EtcdMirror` continuously copies a key range from a source etcd cluster into a +target etcd cluster, one way, as a single supervised stateless pod. Progress is +checkpointed in a reserved, fenced key **in the target etcd** — written in the +same transaction as every applied batch — so the agent has no volume and can be +rescheduled freely. + +It is a **byte-copy of keys and values, not a replica**. If you need a replica, +add members to the cluster; if you need a point-in-time copy with revision +fidelity, use `etcdutl snapshot restore`. + +## What is and is not preserved + +| Property | Preserved? | Notes | +| --- | --- | --- | +| Key names | yes | rewritten by one formula: `key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix)` | +| Values | yes | byte-identical | +| Per-revision atomicity | yes | batches flush only at source-revision boundaries; a source revision is never split across target Txns | +| Revisions / mod_revision / create_revision | **no** | target-assigned. Stored fence tokens, persisted watch bookmarks, and CreateRevision-ordered elections do not survive mirroring | +| Version counters | **no** | target-assigned | +| Leases / TTLs | **no** | leases are stripped; a lease-backed source key becomes a permanent target key. Masked while the mirror runs (source expiry replicates as a delete); at cutover every in-flight leased key is immortal. `status.leaseBackedKeyCount` reports exposure; the cutover runbook includes a purge/re-lease step. Consider `sync.excludePrefixes` for lease-heavy ranges | +| Cross-key Txn atomicity | best-effort | whole revisions are coalesced; a revision larger than `maxTxnOps` is applied as one oversized Txn (provision the target's `--max-txn-ops`) | + +A fidelity-preserving alternative for migrations: seed the target with +`etcdutl snapshot restore --bump-revision --mark-compacted`, then mirror only +the delta using `initialSync.startRevision`. + +## Sync engine behavior + +- **Genesis (InitialSync):** unpinned chunked scan (one byte-bounded page in + flight) with the watch already open from the revision observed before the + scan; buffered events replay over the scanned base. Mid-scan compaction on + the source therefore cannot fail the scan. +- **Steady state:** watch with progress notifications; the checkpoint watermark + advances even when the mirrored prefix is idle. +- **Forced resync** (watch outrun by compaction — e.g. the mirror was down or + paused longer than the source's compaction retention): reported as + `Phase=InitialSync` with condition `Compacted=True/ForcedResync`, bracketed by + `ForcedResyncStarted`/`ForcedResyncCompleted` events. Every forced resync ends + with a mandatory mark-and-sweep prune, so deletes that happened during the + blind window do not resurrect. +- **Retention prerequisite:** source compaction retention must exceed the + worst-case scan + throttled drain time (roughly + `keyCount / min(scanRate, maxOpsPerSecond)`), or genesis/forced resyncs + livelock — surfaced as `ResyncLoopDetected`, which does not self-heal. +- **Checkpoint fencing:** the reserved key carries `{linkUID, epoch, role}` and + every write path (applies, reconciliation repairs, prune deletes) is fenced + with a mod-revision compare, so two agents can never interleave and a + straggler apply after cutover fails loudly. On a failed compare the engine + re-reads the fence — a Txn that committed while its response was lost (WAN + timeout) is recognized as this agent's own write and adopted, never + misreported as a fence violation. +- **Destination overlap guard:** a prune pass that finds ANOTHER link's + reserved fence key inside this link's destination prefix stops with a + permanent prefix-conflict error instead of deleting the sibling mirror's + fence and data. + +## Monitoring / paging algebra + +Page on `Available=False` sustained, **unless** `Compacted=True` and progress +fields are advancing (a forced resync healing itself). `TargetQuotaExhausted` +and `ResyncLoopDetected` page immediately — neither self-heals. + +### Metrics + +The controller serves `etcd_mirror_phase{namespace,name,phase}` and +`etcd_mirror_condition{namespace,name,type,status}` one-hot gauges on the +manager `/metrics` endpoint — updated on every reconcile (including failed +agent `/statusz` polls) and present even when the agent pod never scheduled, +so absent agents still alert. The agent itself exposes the +`etcd_mirror_agent_*` family on its `http` port (8080); scraping it is not +wired by the operator. Shipped assets: +`config/prometheus/etcdmirror_rules.yaml` (PrometheusRule implementing the +paging algebra above, minus the Compacted-and-progressing suppression — that +refinement needs agent scrape, and a suppressed wedged resync must still +page) and `docs/etcdmirror-dashboard.json` (Grafana). + +Never compute lag as `status.sourceRevision - status.lastAppliedRevision`: +revisions are cluster-global, so out-of-prefix source writes inflate the +difference, and the two fields snapshot at different instants. + +The `InitialSyncCompactionRaced` event marks a genesis-scan attempt aborted +and restarted from a fresh revision. One event name, two causes named in the +message — do not conflate them: + +- `WatchBufferOverflow`: the replay buffer exceeded `sync.watchBufferBytes` + before the base scan completed. A memory-bound retry, **not** a compaction + race — raise `watchBufferBytes` or the scan rate for high-churn sources. +- `WatchCompactedMidScan`: a watch reconnect landed below the source compact + revision — the rare genuine race. + +Repeated occurrences of either count toward `ResyncLoopDetected`. + +`InvariantsHeld=True` means the verification invariants hold (lag within +threshold, per-side key counts equal, no drift, pass fresh) — it never means +"safe to cut over"; that is `CutoverReady`, which additionally requires +`spec.mode: Drain` and a reached drain target revision. + +## Operations: error taxonomy and runbook + +| Error | gRPC code | Class | Retry policy | Condition / Reason | +| --- | --- | --- | --- | --- | +| `ErrCompacted` on watch reopen | OutOfRange | Resync | forced resync (scan + mandatory prune); never generic retry | `Compacted=True/ForcedResync`; `forcedResyncCount`++ (`Compacted`) | +| `ErrNoSpace` | ResourceExhausted | Quota | park on slow flat timer; never hot-loop; recovers without genesis once operator compacts/defrags/disarms | `TargetQuotaExhausted=True` (pages immediately) | +| client send cap ("trying to send message larger than max") | ResourceExhausted | Permanent | never retried identically; redacted key surfaced | Failed if unavoidable | +| `ErrTooManyRequests` | ResourceExhausted | Throttle | conservative distinct curve | `TargetThrottled=True` | +| `ErrRequestTooLarge` | InvalidArgument | Permanent | one shrink attempt at revision granularity, else Failed | redacted key surfaced | +| `ErrTooManyOps` | InvalidArgument | Permanent | one shrink attempt at revision granularity; else raise target `--max-txn-ops` or lower `spec.sync.maxTxnOps` | Failed | +| Unavailable / `ErrNoLeader` | Unavailable | Transient | ReconnectBackoff curve | Source/TargetReachable=False (reason NoLeader when applicable) | +| DeadlineExceeded (requestTimeout) | DeadlineExceeded | Transient | ReconnectBackoff — the blackholed-NLB recovery path | reachability, reason RequestTimeout | +| auth token expiry | — | non-issue | clientv3 refreshes transparently | none | +| source version < 3.4 | — | Permanent | never | Failed/`UnsupportedVersion` | +| corrupt/unknown-version checkpoint | — | Permanent (fail closed) | never; operator deletes reserved key | Failed/`CheckpointInvalid` | +| linkUID / cluster-ID mismatch | — | expected transition | genesis + RequireEmpty re-arm | `CheckpointInvalidated` event | +| fence Compare loss (normal apply) | — | optimistic-concurrency loss | re-read, recompute, retry (jitter, not reconnect curve) | internal; persistent genesis-claim loss → FenceError (permanent) | +| N consecutive resyncs, no steady period | — | livelock (meta) | does not self-heal | `ResyncLoopDetected=True` (pages immediately) | + +## Cutover and reversal + +Two-way sync is out of scope permanently: etcd revisions are cluster-local and +there is no per-key provenance channel, so bidirectional sync is structurally +inexpressible without an application-visible format change. + +Cutover (forward): quiesce source writers, set `spec.mode: Drain`, then +`kubectl wait --for=condition=CutoverReady etcdmirror/`. The +`status.cutover` block records the drained revision, verification counts, and +the lease-backed key count for the purge/re-lease step. Once CutoverReady, the +fence key's role is Primary and any straggler mirror write fails its compare. + +Reversal (failback) is delete-and-recreate: delete the CR, create a new one +with swapped endpoints and `initialSync.mode: OverwriteAndPrune` — the +mandatory prune pass removes keys deleted on the new primary since cutover. +Only reverse after the forward mirror reached CutoverReady. + +## Prerequisites (summary) + +- Source etcd >= 3.4 (hard floor, probed at connect); >= 3.4.25 / 3.5.8 + recommended — below that, watch progress notifications are unreliable. +- Target credential range-scoped via etcd RBAC to the effective destination + prefix, **including the reserved checkpoint key** + (default `\x00etcdmirror-checkpoint`; an + override must stay under the effective destination prefix — CEL-enforced). +- Range-defining fields (`source.prefix`, `target.prefix`, `sync.destPrefix`, + `sync.excludePrefixes`, `checkpoint.key`) are immutable: the resume path + never re-scans, so an edited range silently diverges. Change them via + delete-and-recreate with an appropriate `initialSync.mode`. +- Target runs with auto-compaction enabled (resync churn otherwise marches the + default 2GiB quota toward NOSPACE / `TargetQuotaExhausted`). +- Source compaction retention satisfies the formula above for your key count + and rate limit. +- A source running `--client-cert-auth` (RKE1 default) rejects certless + clients at the handshake: supply a client certificate; username/password + auth alone is not viable there. When both are supplied, the token identity + wins and must hold the range-scoped role. +- The agent Deployment runs on the namespace default ServiceAccount with + `automountServiceAccountToken: false`; it needs zero Kubernetes API access, + so no ServiceAccount/Role/RoleBinding is installed for it. diff --git a/go.mod b/go.mod index 6ae8ff14..29d1610b 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( k8s.io/apimachinery v0.36.1 k8s.io/client-go v0.36.1 k8s.io/klog/v2 v2.140.0 + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 sigs.k8s.io/controller-runtime v0.24.1 ) @@ -26,6 +27,7 @@ require ( github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/moby/spdystream v0.5.1 // indirect github.com/vladimirvivien/gexe v0.5.0 // indirect go.etcd.io/raft/v3 v3.6.0 // indirect @@ -33,7 +35,6 @@ require ( go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect k8s.io/streaming v0.36.1 // indirect - k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect sigs.k8s.io/gateway-api v1.5.0 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect @@ -77,7 +78,7 @@ require ( github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect @@ -113,7 +114,7 @@ require ( gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/grpc v1.79.3 // indirect + google.golang.org/grpc v1.79.3 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/internal/controller/etcdmirror_cel_test.go b/internal/controller/etcdmirror_cel_test.go new file mode 100644 index 00000000..56467c5c --- /dev/null +++ b/internal/controller/etcdmirror_cel_test.go @@ -0,0 +1,775 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +// validSourceEndpoint and validTargetEndpoint are baseline endpoints that +// satisfy EtcdMirrorEndpoint's own CEL rules (oneOf, scheme-vs-TLS), so each +// test case only needs to perturb the field(s) it's actually exercising. The +// source endpoint is deliberately scheme-less so TLS blocks can be added or +// removed freely without tripping the scheme rules. +func validSourceEndpoint() ecv1alpha1.EtcdMirrorEndpoint { + return ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: []string{"etcd-source.example.com:2379"}, + Prefix: "/registry/", + } +} + +func validTargetEndpoint() ecv1alpha1.EtcdMirrorEndpoint { + return ecv1alpha1.EtcdMirrorEndpoint{ + ServiceRef: &ecv1alpha1.EtcdMirrorServiceRef{Name: "etcd-target-client"}, + Prefix: "/mirrored/", + } +} + +func newMirror(prefix string, spec ecv1alpha1.EtcdMirrorSpec) *ecv1alpha1.EtcdMirror { + return &ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: prefix, + Namespace: "default", + }, + Spec: spec, + } +} + +// createAndCheck applies the mirror and asserts admission matched wantApply, +// deleting it again on success. +func createAndCheck(t *testing.T, em *ecv1alpha1.EtcdMirror, wantApply bool, msg string) { + t.Helper() + err := k8sClient.Create(t.Context(), em) + if wantApply { + require.NoError(t, err, msg) + _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) + } else { + assert.Error(t, err, msg) + } +} + +// TestEtcdMirrorEndpointOneOfCELValidation drives the endpointList/serviceRef +// exactly-one-of XValidation rule on EtcdMirrorEndpoint, exercised on both +// Source and Target. An empty endpointList is deliberately treated as unset +// (k8s list conventions), so [] alongside a serviceRef must be ACCEPTED. +func TestEtcdMirrorEndpointOneOfCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + neither := ecv1alpha1.EtcdMirrorEndpoint{Prefix: "/x/"} + both := ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: []string{"etcd.example.com:2379"}, + ServiceRef: &ecv1alpha1.EtcdMirrorServiceRef{Name: "etcd-client"}, + } + emptyListOnly := ecv1alpha1.EtcdMirrorEndpoint{EndpointList: []string{}} + emptyListWithServiceRef := ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: []string{}, + ServiceRef: &ecv1alpha1.EtcdMirrorServiceRef{Name: "etcd-client"}, + } + + tests := []struct { + name string + source ecv1alpha1.EtcdMirrorEndpoint + target ecv1alpha1.EtcdMirrorEndpoint + wantApply bool + }{ + { + name: "valid endpointList source, serviceRef target accepted", + source: validSourceEndpoint(), + target: validTargetEndpoint(), + wantApply: true, + }, + { + name: "source with neither endpointList nor serviceRef rejected", + source: neither, + target: validTargetEndpoint(), + wantApply: false, + }, + { + name: "source with both endpointList and serviceRef rejected", + source: both, + target: validTargetEndpoint(), + wantApply: false, + }, + { + name: "source with empty endpointList and no serviceRef rejected", + source: emptyListOnly, + target: validTargetEndpoint(), + wantApply: false, + }, + { + // Pinned deliberately: empty list == unset, so this is the + // serviceRef-only case, not the both-set case. + name: "source with empty endpointList plus serviceRef accepted", + source: emptyListWithServiceRef, + target: validTargetEndpoint(), + wantApply: true, + }, + { + name: "target with neither endpointList nor serviceRef rejected", + source: validSourceEndpoint(), + target: neither, + wantApply: false, + }, + { + name: "target with both endpointList and serviceRef rejected", + source: validSourceEndpoint(), + target: both, + wantApply: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := newMirror("cel-mirror-endpoint-", ecv1alpha1.EtcdMirrorSpec{ + Source: tt.source, + Target: tt.target, + }) + createAndCheck(t, em, tt.wantApply, "endpointList/serviceRef oneOf") + }) + } +} + +// TestEtcdMirrorEndpointSchemeTLSCELValidation drives the two scheme-vs-TLS +// XValidation rules on EtcdMirrorEndpoint: http:// forbids a tls block, +// https:// requires one (an empty tls block means system trust roots). +func TestEtcdMirrorEndpointSchemeTLSCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tlsBlock := &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: &corev1.LocalObjectReference{Name: "etcd-mirror-tls"}, + } + + tests := []struct { + name string + endpoints []string + tls *ecv1alpha1.EtcdMirrorTLS + wantApply bool + }{ + { + name: "https endpoint with tls block accepted", + endpoints: []string{"https://etcd.example.com:2379"}, + tls: tlsBlock, + wantApply: true, + }, + { + name: "https endpoint with empty tls block (system roots) accepted", + endpoints: []string{"https://etcd.example.com:2379"}, + tls: &ecv1alpha1.EtcdMirrorTLS{}, + wantApply: true, + }, + { + name: "https endpoint without tls block rejected", + endpoints: []string{"https://etcd.example.com:2379"}, + tls: nil, + wantApply: false, + }, + { + name: "http endpoint without tls block accepted", + endpoints: []string{"http://etcd.example.com:2379"}, + tls: nil, + wantApply: true, + }, + { + name: "http endpoint with tls block rejected", + endpoints: []string{"http://etcd.example.com:2379"}, + tls: tlsBlock, + wantApply: false, + }, + { + name: "mixed http and https endpoints with tls block rejected", + endpoints: []string{"https://a.example.com:2379", "http://b.example.com:2379"}, + tls: tlsBlock, + wantApply: false, + }, + { + name: "scheme-less endpoint with tls block accepted", + endpoints: []string{"etcd.example.com:2379"}, + tls: tlsBlock, + wantApply: true, + }, + { + name: "scheme-less endpoint without tls block accepted", + endpoints: []string{"etcd.example.com:2379"}, + tls: nil, + wantApply: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: tt.endpoints, + Prefix: "/registry/", + TLS: tt.tls, + } + em := newMirror("cel-mirror-scheme-", ecv1alpha1.EtcdMirrorSpec{ + Source: source, + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "scheme-vs-TLS rule on source") + }) + } + + t.Run("scheme rules also apply to target", func(t *testing.T) { + em := newMirror("cel-mirror-scheme-", ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: []string{"https://etcd-target.example.com:2379"}, + Prefix: "/mirrored/", + }, + }) + createAndCheck(t, em, false, "https target without tls must be rejected") + }) +} + +// TestEtcdMirrorTLSInsecureSkipVerifyCELValidation drives the +// insecureSkipVerify/insecureSkipVerifyAcknowledgeRisk companion-field +// XValidation rule on EtcdMirrorTLS. +func TestEtcdMirrorTLSInsecureSkipVerifyCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + secretRef := &corev1.LocalObjectReference{Name: "etcd-mirror-tls"} + + tests := []struct { + name string + tls *ecv1alpha1.EtcdMirrorTLS + wantApply bool + }{ + { + name: "no TLS block accepted", + tls: nil, + wantApply: true, + }, + { + name: "TLS with verification enabled accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{SecretRef: secretRef}, + wantApply: true, + }, + { + name: "insecureSkipVerify with acknowledgement accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: secretRef, + InsecureSkipVerify: true, + InsecureSkipVerifyAcknowledgeRisk: true, + }, + wantApply: true, + }, + { + name: "insecureSkipVerify with acknowledgement and no secretRef accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{ + InsecureSkipVerify: true, + InsecureSkipVerifyAcknowledgeRisk: true, + }, + wantApply: true, + }, + { + name: "insecureSkipVerify without acknowledgement rejected", + tls: &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: secretRef, + InsecureSkipVerify: true, + }, + wantApply: false, + }, + { + name: "acknowledgement without insecureSkipVerify accepted (not the risky case)", + tls: &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: secretRef, + InsecureSkipVerifyAcknowledgeRisk: true, + }, + wantApply: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := validSourceEndpoint() + source.TLS = tt.tls + em := newMirror("cel-mirror-tls-", ecv1alpha1.EtcdMirrorSpec{ + Source: source, + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "insecureSkipVerify companion rule") + }) + } +} + +// TestEtcdMirrorSecretRefCELValidation drives the secretRef rules: on TLS the +// secretRef is optional (nil = system trust roots) but must have a non-empty +// name when present; on Auth it is required with a non-empty name. +func TestEtcdMirrorSecretRefCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + tls *ecv1alpha1.EtcdMirrorTLS + auth *ecv1alpha1.EtcdMirrorAuth + wantApply bool + }{ + { + name: "TLS secretRef with non-empty name accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{SecretRef: &corev1.LocalObjectReference{Name: "etcd-mirror-tls"}}, + wantApply: true, + }, + { + name: "TLS with nil secretRef (system trust roots) accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{}, + wantApply: true, + }, + { + name: "TLS secretRef with empty name rejected", + tls: &ecv1alpha1.EtcdMirrorTLS{SecretRef: &corev1.LocalObjectReference{Name: ""}}, + wantApply: false, + }, + { + name: "Auth secretRef with non-empty name accepted", + auth: &ecv1alpha1.EtcdMirrorAuth{SecretRef: corev1.LocalObjectReference{Name: "etcd-mirror-auth"}}, + wantApply: true, + }, + { + name: "Auth secretRef with empty name rejected", + auth: &ecv1alpha1.EtcdMirrorAuth{SecretRef: corev1.LocalObjectReference{Name: ""}}, + wantApply: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := validSourceEndpoint() + source.TLS = tt.tls + source.Auth = tt.auth + em := newMirror("cel-mirror-secretref-", ecv1alpha1.EtcdMirrorSpec{ + Source: source, + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "secretRef name rules") + }) + } +} + +// TestEtcdMirrorCABundleRefValidation drives EtcdMirrorCABundleRef's schema +// validation (kind enum, required name). +func TestEtcdMirrorCABundleRefValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + ref *ecv1alpha1.EtcdMirrorCABundleRef + wantApply bool + }{ + { + name: "configMap caBundleRef accepted", + ref: &ecv1alpha1.EtcdMirrorCABundleRef{Kind: "ConfigMap", Name: "mirror-trust"}, + wantApply: true, + }, + { + name: "secret caBundleRef with key accepted", + ref: &ecv1alpha1.EtcdMirrorCABundleRef{Kind: "Secret", Name: "mirror-trust", Key: "bundle.pem"}, + wantApply: true, + }, + { + name: "caBundleRef with empty name rejected", + ref: &ecv1alpha1.EtcdMirrorCABundleRef{Kind: "ConfigMap", Name: ""}, + wantApply: false, + }, + { + name: "caBundleRef with bogus kind rejected", + ref: &ecv1alpha1.EtcdMirrorCABundleRef{Kind: "DaemonSet", Name: "mirror-trust"}, + wantApply: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := validSourceEndpoint() + source.TLS = &ecv1alpha1.EtcdMirrorTLS{CABundleRef: tt.ref} + em := newMirror("cel-mirror-cabundle-", ecv1alpha1.EtcdMirrorSpec{ + Source: source, + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "caBundleRef schema rules") + }) + } +} + +// TestEtcdMirrorInitialSyncCELValidation drives the initialSync rules: the +// mode enum, and the startRevision-requires-Overwrite guard (a target seeded +// via snapshot restore cannot also be required empty). +func TestEtcdMirrorInitialSyncCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + initialSync *ecv1alpha1.EtcdMirrorInitialSyncSpec + wantApply bool + }{ + { + name: "nil initialSync accepted", + initialSync: nil, + wantApply: true, + }, + { + name: "explicit RequireEmpty accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{Mode: ecv1alpha1.EtcdMirrorInitialSyncRequireEmpty}, + wantApply: true, + }, + { + name: "Overwrite accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwrite}, + wantApply: true, + }, + { + name: "OverwriteAndPrune accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwriteAndPrune}, + wantApply: true, + }, + { + name: "bogus mode rejected by enum", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{Mode: "TruncateFirst"}, + wantApply: false, + }, + { + name: "startRevision with Overwrite accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwrite, + StartRevision: 42, + }, + wantApply: true, + }, + { + name: "startRevision with OverwriteAndPrune accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwriteAndPrune, + StartRevision: 42, + }, + wantApply: true, + }, + { + name: "startRevision with explicit RequireEmpty rejected", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncRequireEmpty, + StartRevision: 42, + }, + wantApply: false, + }, + { + name: "startRevision with defaulted mode (RequireEmpty) rejected", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + StartRevision: 42, + }, + wantApply: false, + }, + { + name: "startRevision zero with RequireEmpty accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncRequireEmpty, + StartRevision: 0, + }, + wantApply: true, + }, + { + name: "negative startRevision rejected by minimum", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwrite, + StartRevision: -1, + }, + wantApply: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := newMirror("cel-mirror-initialsync-", ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + InitialSync: tt.initialSync, + }) + createAndCheck(t, em, tt.wantApply, "initialSync rules") + }) + } +} + +// TestEtcdMirrorModeValidation drives the spec.mode enum. +func TestEtcdMirrorModeValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + mode ecv1alpha1.EtcdMirrorMode + wantApply bool + }{ + {name: "mode unset accepted (defaults to Sync)", mode: "", wantApply: true}, + {name: "mode Sync accepted", mode: ecv1alpha1.EtcdMirrorModeSync, wantApply: true}, + {name: "mode Drain accepted", mode: ecv1alpha1.EtcdMirrorModeDrain, wantApply: true}, + {name: "bogus mode rejected", mode: "Bidirectional", wantApply: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := newMirror("cel-mirror-mode-", ecv1alpha1.EtcdMirrorSpec{ + Mode: tt.mode, + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "spec.mode enum") + }) + } +} + +// TestEtcdMirrorImmutabilityCELValidation drives the CEL transition rules: +// source.prefix, target.prefix, sync.destPrefix, sync.excludePrefixes and +// checkpoint.key are immutable; endpoints (and ordinary fields) stay mutable +// — same-cluster endpoint rotation is routine and cross-cluster repoints are +// caught at runtime by the checkpoint's cluster-ID binding, not by spec +// validation. +func TestEtcdMirrorImmutabilityCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + baseSpec := func() ecv1alpha1.EtcdMirrorSpec { + return ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + Sync: ecv1alpha1.EtcdMirrorSyncSpec{ + DestPrefix: "registry/", + ExcludePrefixes: []string{"/registry/events/"}, + }, + Checkpoint: &ecv1alpha1.EtcdMirrorCheckpointSpec{ + // Must live under the effective destination prefix + // (target.prefix "/mirrored/" + destPrefix "registry/"). + Key: "/mirrored/registry/\x00etcdmirror-checkpoint", + }, + } + } + + tests := []struct { + name string + mutate func(em *ecv1alpha1.EtcdMirror) + wantUpdate bool + }{ + { + name: "changing source.prefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Source.Prefix = "/other/" }, + wantUpdate: false, + }, + { + name: "unsetting source.prefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Source.Prefix = "" }, + wantUpdate: false, + }, + { + name: "changing target.prefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Target.Prefix = "/elsewhere/" }, + wantUpdate: false, + }, + { + name: "changing sync.destPrefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Sync.DestPrefix = "moved/" }, + wantUpdate: false, + }, + { + name: "unsetting sync.destPrefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Sync.DestPrefix = "" }, + wantUpdate: false, + }, + { + name: "changing checkpoint.key rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Checkpoint.Key = "/mirrored/registry/\x00other" }, + wantUpdate: false, + }, + { + name: "changing sync.excludePrefixes rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Sync.ExcludePrefixes = []string{"/registry/leases/"} + }, + wantUpdate: false, + }, + { + name: "removing sync.excludePrefixes rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Sync.ExcludePrefixes = nil }, + wantUpdate: false, + }, + { + name: "removing checkpoint block (unsetting key) rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Checkpoint = nil }, + wantUpdate: false, + }, + { + name: "changing source endpoints accepted (endpoints deliberately mutable)", + mutate: func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Source.EndpointList = []string{"etcd-source-b.example.com:2379", "etcd-source-c.example.com:2379"} + }, + wantUpdate: true, + }, + { + name: "changing paused accepted", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Paused = true }, + wantUpdate: true, + }, + { + name: "changing sync.maxOpsPerSecond accepted", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Sync.MaxOpsPerSecond = 250 }, + wantUpdate: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := newMirror("cel-mirror-immutable-", baseSpec()) + require.NoError(t, k8sClient.Create(t.Context(), em), "baseline mirror must be accepted") + defer func() { _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) }() + + tt.mutate(em) + err := k8sClient.Update(t.Context(), em) + if tt.wantUpdate { + assert.NoError(t, err, "update should be accepted") + } else { + assert.Error(t, err, "update should be rejected by transition rule") + } + }) + } + + t.Run("setting sync.destPrefix from unset rejected", func(t *testing.T) { + spec := baseSpec() + spec.Sync = ecv1alpha1.EtcdMirrorSyncSpec{} + spec.Checkpoint = nil + em := newMirror("cel-mirror-immutable-", spec) + require.NoError(t, k8sClient.Create(t.Context(), em)) + defer func() { _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) }() + + em.Spec.Sync.DestPrefix = "late/" + assert.Error(t, k8sClient.Update(t.Context(), em), "unset -> set is still a destPrefix mutation") + }) + + // Present-empty and absent are the same VALUE for these fields (and Go + // typed clients drop "" through omitempty), so a CR created from YAML + // with an explicit destPrefix: "" must stay updatable via typed clients + // — presence-based transition rules would 422 every such update on a + // field it never touched. + t.Run("explicit empty destPrefix stays typed-client updatable", func(t *testing.T) { + u := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "operator.etcd.io/v1alpha1", + "kind": "EtcdMirror", + "metadata": map[string]interface{}{ + "generateName": "cel-mirror-emptydest-", + "namespace": "default", + }, + "spec": map[string]interface{}{ + "source": map[string]interface{}{ + "endpointList": []interface{}{"etcd-source.example.com:2379"}, + "prefix": "/registry/", + }, + "target": map[string]interface{}{ + "serviceRef": map[string]interface{}{"name": "etcd-target-client"}, + "prefix": "/mirrored/", + }, + "sync": map[string]interface{}{ + "destPrefix": "", // stored present-but-empty + }, + }, + }} + require.NoError(t, k8sClient.Create(t.Context(), u), "explicit empty destPrefix must be accepted") + defer func() { _ = k8sClient.Delete(t.Context(), u, &client.DeleteOptions{}) }() + + em := &ecv1alpha1.EtcdMirror{} + require.NoError(t, k8sClient.Get(t.Context(), + client.ObjectKey{Namespace: "default", Name: u.GetName()}, em)) + em.Spec.Paused = true // typed round-trip drops destPrefix to absent + assert.NoError(t, k8sClient.Update(t.Context(), em), + "a typed-client update must not be rejected for a field it never touched") + }) +} + +// TestEtcdMirrorCheckpointKeyCELValidation drives the create-time rule that +// checkpoint.key must live under the effective destination prefix: the +// engine rejects anything else permanently at first start, and the key's own +// immutability would otherwise make the Failed CR unrepairable in place. +func TestEtcdMirrorCheckpointKeyCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + spec ecv1alpha1.EtcdMirrorSpec + wantApply bool + }{ + { + name: "key under effective destination prefix accepted", + spec: ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + Checkpoint: &ecv1alpha1.EtcdMirrorCheckpointSpec{ + Key: "/mirrored/\x00my-checkpoint", + }, + }, + wantApply: true, + }, + { + name: "key outside destination prefix rejected", + spec: ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + Checkpoint: &ecv1alpha1.EtcdMirrorCheckpointSpec{ + Key: "/checkpoints/mirror-a", + }, + }, + wantApply: false, + }, + { + name: "key under target.prefix but outside destPrefix rejected", + spec: ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + Sync: ecv1alpha1.EtcdMirrorSyncSpec{DestPrefix: "registry/"}, + Checkpoint: &ecv1alpha1.EtcdMirrorCheckpointSpec{ + Key: "/mirrored/\x00etcdmirror-checkpoint", + }, + }, + wantApply: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := newMirror("cel-mirror-ckptkey-", tt.spec) + createAndCheck(t, em, tt.wantApply, "checkpoint.key placement") + }) + } +} diff --git a/internal/controller/etcdmirror_controller.go b/internal/controller/etcdmirror_controller.go new file mode 100644 index 00000000..8fea27b2 --- /dev/null +++ b/internal/controller/etcdmirror_controller.go @@ -0,0 +1,717 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "errors" + "fmt" + "net" + "strconv" + "strings" + "sync" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + "go.etcd.io/etcd-operator/pkg/mirroragent" +) + +const ( + // checkpointCleanupFinalizer guards deletion of the reserved checkpoint + // key in the TARGET etcd (the CR's only externally-persisted state). + checkpointCleanupFinalizer = "operator.etcd.io/checkpoint-cleanup" + // skipCheckpointCleanupAnnotation is the documented escape hatch: set to + // "true" to let deletion proceed without reaching the target (the + // reserved key is orphaned and must be removed by hand). + skipCheckpointCleanupAnnotation = "operator.etcd.io/skip-checkpoint-cleanup" + + // certExpiryLeadWindow: a referenced certificate expiring within this + // window draws a Warning event. cert-manager renews ~30d out, so <14d + // remaining means the renewal machinery has already failed. + certExpiryLeadWindow = 14 * 24 * time.Hour + // warningDampInterval bounds re-emission of standing warnings + // (cert expiry, insecureSkipVerify, cleanup failures) per CR. In-memory: + // a controller restart re-emits early, which is the conservative side. + warningDampInterval = 12 * time.Hour + + // finalizer pacing: wait for agent pods to terminate, then retry failed + // checkpoint deletes with a bounded exponential backoff. + finalizerPodWait = 5 * time.Second + finalizerRetryInitial = 30 * time.Second + finalizerRetryMax = 5 * time.Minute + validationRetryBackoff = 30 * time.Second + + eventActionReconcile = "Reconcile" + eventActionFinalize = "Finalize" +) + +// EtcdMirrorReconciler reconciles an EtcdMirror into a size-1 stateless +// mirror-agent Deployment and mirrors the agent's /statusz snapshot into +// status. StatusClient and Cleaner are injectable seams: envtest has no +// kubelet, so agent pods never run there and neither /statusz nor the target +// etcd is reachable — tests substitute fakes and drive Reconcile directly. +type EtcdMirrorReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder events.EventRecorder + // AgentImage is the image agent Deployments run (the operator image + // itself; the binary ships at /mirror-agent). Unset leaves EtcdMirror CRs + // Pending with reason AgentImageNotConfigured. + AgentImage string + // StatusClient polls agent pods' /statusz. Defaulted in SetupWithManager. + StatusClient AgentStatusClient + // Cleaner deletes the reserved checkpoint key during finalization. + // Defaulted in SetupWithManager. + Cleaner CheckpointCleaner + + // In-memory ledgers (lost on restart, which is acceptable: the lag window + // restarts, standing warnings conservatively re-emit, and counter bases + // re-derive from persisted status). + mu sync.Mutex + // lagSince: when the watermark gap first exceeded the lag threshold. + lagSince map[types.UID]time.Time + // warnedAt: last emission per damped-warning key ("/"). + warnedAt map[string]time.Time + // counterBases: per-CR offsets rebasing the agent's process-local + // monotonic counters onto persisted status across pod restarts. + counterBases map[types.UID]agentCounterBase +} + +// agentCounterBase carries the persisted-counter offset for one agent pod so +// forcedResyncCount/scanRestartCount never regress when the pod restarts (the +// status contract declares both "Monotonic, never reset"; the agent's copies +// are process memory). +type agentCounterBase struct { + podUID types.UID + forcedResync int64 + scanRestart int64 +} + +// +kubebuilder:rbac:groups=operator.etcd.io,resources=etcdmirrors,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=operator.etcd.io,resources=etcdmirrors/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=operator.etcd.io,resources=etcdmirrors/finalizers,verbs=update +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch +// The manager's recorder emits events.k8s.io/v1 Events; the core-group grant +// alone silently drops them on a real apiserver (invisible in envtest). +// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch + +// Reconcile drives one EtcdMirror: finalize on delete, otherwise validate, +// guard, render the agent Deployment, poll /statusz, and mirror the snapshot +// into status. +func (r *EtcdMirrorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + em := &ecv1alpha1.EtcdMirror{} + if err := r.Get(ctx, req.NamespacedName, em); err != nil { + if apierrors.IsNotFound(err) { + // Covers deletions observed after finalizer removal and + // controller-restart races: no stale series for a gone CR. + deleteEtcdMirrorMetrics(req.Namespace, req.Name) + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + if !em.DeletionTimestamp.IsZero() { + // Deletion is irreversible, so drop the series now: finalization can + // retry checkpoint cleanup for an unbounded window (unreachable + // target), during which frozen pre-delete gauges would either keep + // paging for an intentionally deleted mirror or keep reporting + // Available=true for an agent already gone. Idempotent across + // finalize retries; the NotFound and removeFinalizer deletes remain + // as backstops. + deleteEtcdMirrorMetrics(em.Namespace, em.Name) + return r.finalize(ctx, em) + } + + if !controllerutil.ContainsFinalizer(em, checkpointCleanupFinalizer) { + controllerutil.AddFinalizer(em, checkpointCleanupFinalizer) + if err := r.Update(ctx, em); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + + prior := em.Status.DeepCopy() + res, err := r.reconcileMirror(ctx, em, prior) + if !apiequality.Semantic.DeepEqual(prior, &em.Status) { + if statusErr := r.Status().Update(ctx, em); statusErr != nil { + logger.Error(statusErr, "failed to update EtcdMirror status") + if err == nil { + err = statusErr + } + } + } + // Unconditionally: every reconcile outcome keeps the gauges honest, + // including failed /statusz polls (which set Available=Unknown above). + updateEtcdMirrorMetrics(em) + return res, err +} + +// reconcileMirror is the non-deleting path. It mutates em.Status; the caller +// persists it. +func (r *EtcdMirrorReconciler) reconcileMirror( + ctx context.Context, em *ecv1alpha1.EtcdMirror, prior *ecv1alpha1.EtcdMirrorStatus, +) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + // Standing spec warnings (cert expiry lead window, insecureSkipVerify) + // are spec/Secret-derived and independent of agent state: emit on every + // non-finalizing reconcile so Pending/Paused/guard-blocked mirrors warn + // too (the InsecureSkipVerify contract has no reachability qualifier). + r.emitSpecWarnings(ctx, em, time.Now()) + + // Validate: agent image, credential Secrets, serviceRef resolution. Any + // failure is an environment/spec problem, not an agent failure: Pending. + if r.AgentImage == "" { + r.blockValidation(ctx, em, prior, reasonAgentImageNotConfigured, + "the operator was started without --mirror-agent-image; EtcdMirror CRs stay Pending until it is set") + return ctrl.Result{RequeueAfter: validationRetryBackoff}, nil + } + in := agentWorkloadInput{image: r.AgentImage} + var err error + if in.sourceCreds, err = resolveSideCreds(ctx, r.Client, logger, em.Namespace, sideSourceName, em.Spec.Source); err == nil { + in.targetCreds, err = resolveSideCreds(ctx, r.Client, logger, em.Namespace, sideTargetName, em.Spec.Target) + } + if err == nil { + if in.sourceEndpoints, err = r.endpointsForSide(ctx, em, em.Spec.Source); err == nil { + in.targetEndpoints, err = r.endpointsForSide(ctx, em, em.Spec.Target) + } + } + if err != nil { + var ce *credsError + if errors.As(err, &ce) { + r.blockValidation(ctx, em, prior, ce.Reason, ce.Error()) + return ctrl.Result{RequeueAfter: validationRetryBackoff}, nil + } + return ctrl.Result{}, err + } + + // Guards: overlapping destination ranges and two-way loops. The loser + // (the newer CR) is stopped; the conflict clears when the sibling goes. + blocked, res, err := r.applyGuards(ctx, em, prior, in) + if err != nil || blocked { + return res, err + } + + // Paused: scale to zero, keep the rest of status honest (untouched). + if em.Spec.Paused { + if _, err := r.applyAgentDeployment(ctx, em, in, 0); err != nil { + return ctrl.Result{}, err + } + em.Status.Phase = ecv1alpha1.EtcdMirrorPhasePaused + em.Status.ObservedGeneration = em.Generation + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, + reasonPaused, "spec.paused is true; the agent Deployment is scaled to zero (the checkpoint is retained)") + return ctrl.Result{RequeueAfter: slowRequeueInterval}, nil + } + + // Render and apply the size-1 Deployment. + created, err := r.applyAgentDeployment(ctx, em, in, 1) + if err != nil { + return ctrl.Result{}, err + } + if created { + em.Status.Phase = ecv1alpha1.EtcdMirrorPhasePending + em.Status.ObservedGeneration = em.Generation + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, + reasonAgentPodNotReady, "agent Deployment created; waiting for its pod") + return ctrl.Result{RequeueAfter: statusPollInterval}, nil + } + + // Locate the agent pod. Never poll /statusz without a running pod IP. + pod, err := r.findAgentPod(ctx, em) + if err != nil { + return ctrl.Result{}, err + } + if pod == nil || pod.Status.Phase != corev1.PodRunning || pod.Status.PodIP == "" { + if pod != nil { + em.Status.AgentPod = pod.Name + } + if em.Status.Phase == "" { + em.Status.Phase = ecv1alpha1.EtcdMirrorPhasePending + } + em.Status.ObservedGeneration = em.Generation + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, + reasonAgentPodNotReady, "agent pod is not running yet") + return ctrl.Result{RequeueAfter: statusPollInterval}, nil + } + em.Status.AgentPod = pod.Name + + // Poll /statusz through the seam. + snap, err := r.StatusClient.Snapshot(ctx, net.JoinHostPort(pod.Status.PodIP, strconv.Itoa(agentHTTPPort))) + if err != nil { + // ALL prior status fields retained — including phase: Degraded is + // reserved for the agent's own retry/backoff loop and a poll failure + // says nothing about the agent (a controller-side route blip must not + // flip a healthy mirror, nor mask a terminal Failed). Staleness stays + // observable via LastStatusSyncTime not advancing. + reason := reasonAgentStatusUnreachable + var decodeErr *snapshotDecodeError + if errors.As(err, &decodeErr) { + reason = reasonSnapshotDecodeFailed + } + em.Status.ObservedGeneration = em.Generation + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionUnknown, + reason, fmt.Sprintf("polling agent /statusz: %v", err)) + return ctrl.Result{RequeueAfter: statusPollInterval}, nil + } + + // Rebase the process-local monotonic counters before anything reads them. + r.adjustSnapshotCounters(em, prior, pod.UID, snap) + + // Events first (the persisted counters/conditions are the dedup ledger, + // so emit before the snapshot overwrites them), then the mapping. + r.emitSnapshotEvents(em, prior, snap) + now := time.Now() + r.mu.Lock() + if r.lagSince == nil { + r.lagSince = make(map[types.UID]time.Time) + } + lagSince := r.lagSince[em.UID] + r.mu.Unlock() + lagSince = applySnapshotToStatus(em, snap, now, lagSince) + r.mu.Lock() + if lagSince.IsZero() { + delete(r.lagSince, em.UID) + } else { + r.lagSince[em.UID] = lagSince + } + r.mu.Unlock() + + if em.Status.Phase == ecv1alpha1.EtcdMirrorPhaseFailed { + return ctrl.Result{RequeueAfter: slowRequeueInterval}, nil + } + return ctrl.Result{RequeueAfter: statusPollInterval}, nil +} + +// blockValidation parks the CR on a validation failure with a specific +// Available reason, emitting one Warning per transition into that reason. +// Phase drops to Pending ("the agent workload has not been created yet") only +// when that is true: a Deployment rendered before the failure (a referenced +// Secret/Service deleted later, an operator restart without the image flag) +// keeps its pod running on mounted material, so the prior phase is retained +// and only the condition carries the failure — status is stale until +// validation passes, observable via LastStatusSyncTime. +func (r *EtcdMirrorReconciler) blockValidation( + ctx context.Context, em *ecv1alpha1.EtcdMirror, prior *ecv1alpha1.EtcdMirrorStatus, reason, message string, +) { + dep := &appsv1.Deployment{} + err := r.Get(ctx, types.NamespacedName{Namespace: em.Namespace, Name: deploymentNameForEtcdMirror(em)}, dep) + if err != nil || em.Status.Phase == "" { + em.Status.Phase = ecv1alpha1.EtcdMirrorPhasePending + } else { + message += "; the existing agent Deployment keeps running on previously mounted material" + + " and its status mirror is stale until validation passes" + } + em.Status.ObservedGeneration = em.Generation + priorCond := meta.FindStatusCondition(prior.Conditions, ecv1alpha1.EtcdMirrorConditionAvailable) + if priorCond == nil || priorCond.Reason != reason { + r.Recorder.Eventf(em, nil, corev1.EventTypeWarning, reason, eventActionReconcile, "%s", message) + } + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, reason, message) +} + +// adjustSnapshotCounters rebases the agent's process-local monotonic counters +// onto the persisted status so a pod restart never regresses them (nor +// re-fires the counter-keyed events). First sight of a pod assumes the +// persisted counters already include the snapshot's — true after a controller +// restart, and a fresh pod reports 0; increments that land between pod start +// and its first poll are absorbed into the base. The max clamps cover a +// container restart inside one pod. +func (r *EtcdMirrorReconciler) adjustSnapshotCounters( + em *ecv1alpha1.EtcdMirror, prior *ecv1alpha1.EtcdMirrorStatus, podUID types.UID, snap *mirroragent.Snapshot, +) { + r.mu.Lock() + if r.counterBases == nil { + r.counterBases = make(map[types.UID]agentCounterBase) + } + b, ok := r.counterBases[em.UID] + if !ok || b.podUID != podUID { + b = agentCounterBase{ + podUID: podUID, + forcedResync: max(int64(prior.ForcedResyncCount)-snap.ForcedResyncCount, 0), + scanRestart: max(prior.ScanRestartCount-snap.ScanRestartCount, 0), + } + r.counterBases[em.UID] = b + } + r.mu.Unlock() + snap.ForcedResyncCount = max(b.forcedResync+snap.ForcedResyncCount, int64(prior.ForcedResyncCount)) + snap.ScanRestartCount = max(b.scanRestart+snap.ScanRestartCount, prior.ScanRestartCount) +} + +// endpointsForSide resolves one side to the comma-joined ---endpoints +// value: the endpointList verbatim, or the serviceRef's DNS name with a +// resolved numeric port. +func (r *EtcdMirrorReconciler) endpointsForSide( + ctx context.Context, em *ecv1alpha1.EtcdMirror, ep ecv1alpha1.EtcdMirrorEndpoint, +) (string, error) { + if len(ep.EndpointList) > 0 { + return strings.Join(ep.EndpointList, ","), nil + } + if ep.ServiceRef != nil { + return serviceRefEndpoints(ctx, r.Client, em.Namespace, ep.ServiceRef) + } + return "", &credsError{Reason: reasonInvalidConfig, msg: "endpoint has neither endpointList nor serviceRef"} +} + +// applyGuards evaluates PrefixConflict and DirectionConflict against every +// other EtcdMirror. Returns blocked=true when em is the conflict loser (its +// Deployment is scaled to zero and the CR parks in Pending). +func (r *EtcdMirrorReconciler) applyGuards( + ctx context.Context, em *ecv1alpha1.EtcdMirror, prior *ecv1alpha1.EtcdMirrorStatus, in agentWorkloadInput, +) (bool, ctrl.Result, error) { + all := &ecv1alpha1.EtcdMirrorList{} + if err := r.List(ctx, all); err != nil { + return false, ctrl.Result{}, err + } + + prefixConflict := findPrefixConflict(em, all.Items) + directionConflict := findDirectionConflict(em, all.Items) + + // Both conditions are (re)evaluated on every reconcile — even when the + // other guard blocks — so a True left by a since-deleted sibling clears. + // DirectionConflict is a mutual property: True on both CRs. + if directionConflict != nil { + r.setConflictCondition(em, prior, directionConflict) + } else { + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionDirectionConflict, metav1.ConditionFalse, + reasonNoConflict, "no other EtcdMirror mirrors the opposite direction between these clusters") + } + // PrefixConflict: only the loser reports True (the winner keeps its + // range), but the winner's False names the parked sibling honestly. + switch { + case prefixConflict == nil: + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionPrefixConflict, metav1.ConditionFalse, + reasonNoConflict, "no other EtcdMirror overlaps this effective destination range on the same target cluster") + case !isConflictLoser(em, prefixConflict.sibling): + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionPrefixConflict, metav1.ConditionFalse, + reasonConflictWinner, fmt.Sprintf( + "EtcdMirror %s/%s overlaps this effective destination range but is the newer CR: it is parked, this mirror keeps the range", + prefixConflict.sibling.Namespace, prefixConflict.sibling.Name)) + prefixConflict = nil + } + + if prefixConflict != nil { + return true, r.blockOnConflict(ctx, em, prior, in, prefixConflict), nil + } + if directionConflict != nil && isConflictLoser(em, directionConflict.sibling) { + return true, r.blockOnConflict(ctx, em, prior, in, nil), nil + } + return false, ctrl.Result{}, nil +} + +// setConflictCondition raises a guard condition, emitting one Warning per +// False->True transition. +func (r *EtcdMirrorReconciler) setConflictCondition( + em *ecv1alpha1.EtcdMirror, prior *ecv1alpha1.EtcdMirrorStatus, c *mirrorConflict, +) { + if !meta.IsStatusConditionTrue(prior.Conditions, c.conditionType) { + r.Recorder.Eventf(em, nil, corev1.EventTypeWarning, c.conditionType, eventActionReconcile, "%s", c.message) + } + setMirrorCondition(em, c.conditionType, metav1.ConditionTrue, reasonConflict, c.message) +} + +// blockOnConflict stops the conflict loser: condition True, Deployment scaled +// to zero, Phase=Pending (the conflict clears when the sibling is deleted, so +// this is not Failed). +func (r *EtcdMirrorReconciler) blockOnConflict( + ctx context.Context, em *ecv1alpha1.EtcdMirror, prior *ecv1alpha1.EtcdMirrorStatus, + in agentWorkloadInput, c *mirrorConflict, +) ctrl.Result { + condType := ecv1alpha1.EtcdMirrorConditionDirectionConflict + if c != nil { + r.setConflictCondition(em, prior, c) + condType = c.conditionType + } + if _, err := r.applyAgentDeployment(ctx, em, in, 0); err != nil { + log.FromContext(ctx).Error(err, "failed to scale conflicting agent Deployment to zero") + } + em.Status.Phase = ecv1alpha1.EtcdMirrorPhasePending + em.Status.ObservedGeneration = em.Generation + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, + condType, "stopped by a "+condType+" with another EtcdMirror; see that condition") + return ctrl.Result{RequeueAfter: slowRequeueInterval} +} + +// applyAgentDeployment renders and CreateOrPatches the agent Deployment. +// Returns whether it was created this reconcile. +func (r *EtcdMirrorReconciler) applyAgentDeployment( + ctx context.Context, em *ecv1alpha1.EtcdMirror, in agentWorkloadInput, replicas int32, +) (bool, error) { + desired := renderAgentDeployment(em, in, replicas) + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: desired.Name, Namespace: desired.Namespace}, + } + op, err := controllerutil.CreateOrPatch(ctx, r.Client, dep, func() error { + dep.Labels = desired.Labels + dep.Spec = desired.Spec + return controllerutil.SetControllerReference(em, dep, r.Scheme) + }) + if err != nil { + return false, fmt.Errorf("applying agent Deployment: %w", err) + } + return op == controllerutil.OperationResultCreated, nil +} + +// findAgentPod returns the newest non-terminating pod of the agent +// Deployment, or nil. +func (r *EtcdMirrorReconciler) findAgentPod(ctx context.Context, em *ecv1alpha1.EtcdMirror) (*corev1.Pod, error) { + pods := &corev1.PodList{} + if err := r.List(ctx, pods, + client.InNamespace(em.Namespace), + client.MatchingLabels(etcdMirrorAgentLabels(em))); err != nil { + return nil, err + } + var newest *corev1.Pod + for i := range pods.Items { + p := &pods.Items[i] + if !p.DeletionTimestamp.IsZero() { + continue + } + if newest == nil || newest.CreationTimestamp.Before(&p.CreationTimestamp) { + newest = p + } + } + return newest, nil +} + +// emitSnapshotEvents emits the operator-facing events whose dedup ledger is +// the persisted status (counters and prior conditions) — must run BEFORE +// applySnapshotToStatus overwrites those fields. Steady state emits nothing. +func (r *EtcdMirrorReconciler) emitSnapshotEvents( + em *ecv1alpha1.EtcdMirror, prior *ecv1alpha1.EtcdMirrorStatus, snap *mirroragent.Snapshot, +) { + // Forced resyncs advanced since the last persisted count. Multiple + // resyncs between polls collapse into one event pair — accepted: the + // counter still records them all. + if snap.ForcedResyncCount > int64(prior.ForcedResyncCount) { + r.Recorder.Eventf(em, nil, corev1.EventTypeWarning, + ecv1alpha1.EtcdMirrorEventForcedResyncStarted, eventActionReconcile, + "forced resync #%d started (trigger: %s)", snap.ForcedResyncCount, snap.LastResyncReason) + if snap.LastResyncReason == mirroragent.ResyncReasonClusterIDMismatch { + r.Recorder.Eventf(em, nil, corev1.EventTypeWarning, + ecv1alpha1.EtcdMirrorEventCheckpointInvalidated, eventActionReconcile, + "checkpoint discarded: a bound cluster ID no longer matches; forcing genesis and re-arming the RequireEmpty check") + } + } + if meta.IsStatusConditionTrue(prior.Conditions, ecv1alpha1.EtcdMirrorConditionCompacted) && + !snap.Compacted && snap.Phase == mirroragent.PhaseSyncing { + r.Recorder.Eventf(em, nil, corev1.EventTypeNormal, + ecv1alpha1.EtcdMirrorEventForcedResyncCompleted, eventActionReconcile, + "forced resync completed; steady-state syncing resumed") + } + if snap.ScanRestartCount > prior.ScanRestartCount { + r.Recorder.Eventf(em, nil, corev1.EventTypeWarning, + ecv1alpha1.EtcdMirrorEventInitialSyncCompactionRaced, eventActionReconcile, + "genesis scan attempt aborted and restarted from a fresh R0 (restart #%d, cause: %s)", + snap.ScanRestartCount, snap.LastScanRestartCause) + } + newViolation := snap.Phase == mirroragent.PhaseFailed && snap.LastErrorReason == "EmptyTargetViolation" + if newViolation && !meta.IsStatusConditionTrue(prior.Conditions, ecv1alpha1.EtcdMirrorConditionEmptyTargetViolation) { + r.Recorder.Eventf(em, nil, corev1.EventTypeWarning, + ecv1alpha1.EtcdMirrorConditionEmptyTargetViolation, eventActionReconcile, + "destination prefix was non-empty at genesis; to clear it run: %s", + etcdctlDelCommand(effectiveDestPrefix(em))) + } +} + +// emitSpecWarnings emits the standing (12h-damped) warnings derived from the +// spec and referenced Secrets: certificates in the expiry lead window and +// insecureSkipVerify. +func (r *EtcdMirrorReconciler) emitSpecWarnings(ctx context.Context, em *ecv1alpha1.EtcdMirror, now time.Time) { + for _, side := range []struct { + name string + ep ecv1alpha1.EtcdMirrorEndpoint + }{{sideSourceName, em.Spec.Source}, {sideTargetName, em.Spec.Target}} { + for _, exp := range sideCertExpiries(ctx, r.Client, em.Namespace, side.name, side.ep) { + if exp.NotAfter.Sub(now) < certExpiryLeadWindow { + r.dampedEventf(em, now, "certexpiry/"+exp.Side+"/"+exp.Kind, corev1.EventTypeWarning, + ecv1alpha1.EtcdMirrorEventCertificateExpiringSoon, + "%s %s expires %s (within the %s lead window); the renewal machinery appears to have failed", + exp.Side, exp.Kind, exp.NotAfter.UTC().Format(time.RFC3339), certExpiryLeadWindow) + } + } + if side.ep.TLS != nil && side.ep.TLS.InsecureSkipVerify { + r.dampedEventf(em, now, "insecureskipverify/"+side.name, corev1.EventTypeWarning, + ecv1alpha1.EtcdMirrorEventInsecureSkipVerifyEnabled, + "%s TLS verification is disabled (insecureSkipVerify: true) — strongly discouraged", side.name) + } + } +} + +// dampedEventf emits at most one event per (CR, key) per warningDampInterval. +func (r *EtcdMirrorReconciler) dampedEventf( + em *ecv1alpha1.EtcdMirror, now time.Time, key, eventtype, reason, note string, args ...any, +) { + ledgerKey := string(em.UID) + "/" + key + r.mu.Lock() + if r.warnedAt == nil { + r.warnedAt = make(map[string]time.Time) + } + if last, ok := r.warnedAt[ledgerKey]; ok && now.Sub(last) < warningDampInterval { + r.mu.Unlock() + return + } + r.warnedAt[ledgerKey] = now + r.mu.Unlock() + r.Recorder.Eventf(em, nil, eventtype, reason, eventActionReconcile, note, args...) +} + +// finalize handles CR deletion: stop the agent, delete the reserved +// checkpoint key from the target (through the Cleaner seam), then release the +// finalizer. The skip-checkpoint-cleanup annotation is the escape hatch for +// permanently unreachable targets. +func (r *EtcdMirrorReconciler) finalize(ctx context.Context, em *ecv1alpha1.EtcdMirror) (ctrl.Result, error) { + logger := log.FromContext(ctx) + if !controllerutil.ContainsFinalizer(em, checkpointCleanupFinalizer) { + return ctrl.Result{}, nil + } + + if em.Annotations[skipCheckpointCleanupAnnotation] == "true" { + r.Recorder.Eventf(em, nil, corev1.EventTypeNormal, "CheckpointCleanupSkipped", eventActionFinalize, + "checkpoint cleanup skipped by the %s annotation; reserved key %q remains in the target etcd", + skipCheckpointCleanupAnnotation, checkpointKeyForMirror(em)) + return r.removeFinalizer(ctx, em) + } + + // The agent must stop before the key is deleted, or its next fenced Txn + // recreates it: delete the Deployment, wait for its pods to be gone. + dep := &appsv1.Deployment{} + err := r.Get(ctx, types.NamespacedName{Namespace: em.Namespace, Name: deploymentNameForEtcdMirror(em)}, dep) + switch { + case err == nil: + if dep.DeletionTimestamp.IsZero() { + if err := r.Delete(ctx, dep); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + case !apierrors.IsNotFound(err): + return ctrl.Result{}, err + } + pods := &corev1.PodList{} + if err := r.List(ctx, pods, + client.InNamespace(em.Namespace), + client.MatchingLabels(etcdMirrorAgentLabels(em))); err != nil { + return ctrl.Result{}, err + } + if len(pods.Items) > 0 { + logger.Info("waiting for agent pods to terminate before checkpoint cleanup", "pods", len(pods.Items)) + return ctrl.Result{RequeueAfter: finalizerPodWait}, nil + } + + var skipReason string + tgt, err := resolveFinalizerTarget(ctx, r.Client, logger, em) + if err == nil { + skipReason, err = r.Cleaner.DeleteCheckpoint(ctx, tgt) + } + if err != nil { + // Bounded-backoff retries forever; the annotation is the exit. + r.dampedEventf(em, time.Now(), "checkpointcleanup", corev1.EventTypeWarning, + "CheckpointCleanupFailed", + "deleting reserved checkpoint key from the target failed (will retry; set the %s annotation to skip): %v", + skipCheckpointCleanupAnnotation, err) + backoff := finalizerRetryBackoff(time.Since(em.DeletionTimestamp.Time)) + logger.Error(err, "checkpoint cleanup failed", "retryAfter", backoff) + return ctrl.Result{RequeueAfter: backoff}, nil + } + if skipReason != "" { + // A foreign or undecodable fence is provably not this CR's state + // (e.g. this CR was a parked PrefixConflict loser and the key is the + // winner's live fence): nothing of ours to clean, deletion proceeds. + r.Recorder.Eventf(em, nil, corev1.EventTypeNormal, "CheckpointNotOwned", eventActionFinalize, + "reserved key %q left in place: %s", tgt.Key, skipReason) + } + return r.removeFinalizer(ctx, em) +} + +func (r *EtcdMirrorReconciler) removeFinalizer(ctx context.Context, em *ecv1alpha1.EtcdMirror) (ctrl.Result, error) { + controllerutil.RemoveFinalizer(em, checkpointCleanupFinalizer) + if err := r.Update(ctx, em); err != nil { + return ctrl.Result{}, err + } + deleteEtcdMirrorMetrics(em.Namespace, em.Name) + r.mu.Lock() + delete(r.lagSince, em.UID) + delete(r.counterBases, em.UID) + prefix := string(em.UID) + "/" + for k := range r.warnedAt { + if strings.HasPrefix(k, prefix) { + delete(r.warnedAt, k) + } + } + r.mu.Unlock() + return ctrl.Result{}, nil +} + +// finalizerRetryBackoff doubles from finalizerRetryInitial as elapsed +// deletion time accumulates, capped at finalizerRetryMax — bounded frequency, +// never gives up silently. +func finalizerRetryBackoff(elapsed time.Duration) time.Duration { + backoff := finalizerRetryInitial + remaining := elapsed + for backoff < finalizerRetryMax && remaining >= backoff { + remaining -= backoff + backoff *= 2 + } + return min(backoff, finalizerRetryMax) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *EtcdMirrorReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Recorder = mgr.GetEventRecorder("etcdmirror-controller") + if r.StatusClient == nil { + r.StatusClient = newHTTPAgentStatusClient() + } + if r.Cleaner == nil { + r.Cleaner = etcdCheckpointCleaner{} + } + // The controller's own status writes must not re-enqueue the CR: every + // healthy poll advances LastStatusSyncTime, so an unfiltered watch turns + // the poll cadence into a self-triggering hot loop. Generation covers + // spec changes and deletion; annotations keep the skip-checkpoint-cleanup + // escape hatch responsive. Poll cadence rests on the returned + // RequeueAfter values, which is the design. + return ctrl.NewControllerManagedBy(mgr). + For(&ecv1alpha1.EtcdMirror{}, builder.WithPredicates(predicate.Or( + predicate.GenerationChangedPredicate{}, + predicate.AnnotationChangedPredicate{}, + ))). + Owns(&appsv1.Deployment{}). + Complete(r) +} diff --git a/internal/controller/etcdmirror_controller_test.go b/internal/controller/etcdmirror_controller_test.go new file mode 100644 index 00000000..97078395 --- /dev/null +++ b/internal/controller/etcdmirror_controller_test.go @@ -0,0 +1,1014 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "math/big" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + "go.etcd.io/etcd-operator/pkg/mirroragent" +) + +func getAgentDeployment(t *testing.T, em *ecv1alpha1.EtcdMirror) *appsv1.Deployment { + t.Helper() + dep := &appsv1.Deployment{} + err := k8sClient.Get(t.Context(), + types.NamespacedName{Namespace: em.Namespace, Name: deploymentNameForEtcdMirror(em)}, dep) + require.NoError(t, err) + return dep +} + +func TestEtcdMirrorReconcile_RendersDeployment(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{snap: healthySnapshot()} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + ns := createTestNamespace(t) + em := newEnvtestMirror(t, ns, nil) + + // First reconcile adds the finalizer. + reconcileMirrorOnce(t, r, em) + em = refreshMirror(t, em) + assert.Contains(t, em.Finalizers, checkpointCleanupFinalizer) + + // Second reconcile creates the Deployment; Pending before any pod. + res := reconcileMirrorOnce(t, r, em) + assert.Equal(t, statusPollInterval, res.RequeueAfter) + em = refreshMirror(t, em) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhasePending, em.Status.Phase) + + dep := getAgentDeployment(t, em) + require.Len(t, dep.OwnerReferences, 1) + assert.Equal(t, em.Name, dep.OwnerReferences[0].Name) + assert.True(t, *dep.OwnerReferences[0].Controller) + c := dep.Spec.Template.Spec.Containers[0] + assert.Equal(t, []string{"/mirror-agent"}, c.Command) + assert.Equal(t, testAgentImage, c.Image) + assert.Contains(t, c.Args, "--link-uid="+string(em.UID)) + assert.Contains(t, c.Args, "--epoch=1") + assert.Contains(t, c.Args, "--mode=Sync") + assert.Contains(t, c.Args, "--source-prefix=/registry/") + assert.Contains(t, c.Args, "--source-endpoints="+em.Spec.Source.EndpointList[0]) + assert.Contains(t, c.Args, "--target-endpoints="+em.Spec.Target.EndpointList[0]) + + // No pod yet: AgentPodNotReady, no statusz call. + reconcileMirrorOnce(t, r, em) + em = refreshMirror(t, em) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, reasonAgentPodNotReady) + assert.Zero(t, sc.callCount()) + + // Pod running with an IP: the snapshot lands in status. + pod := makeAgentPodReady(t, em) + res = reconcileMirrorOnce(t, r, em) + assert.Equal(t, statusPollInterval, res.RequeueAfter) + em = refreshMirror(t, em) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhaseSyncing, em.Status.Phase) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionTrue, reasonWatermarkAdvancing) + assert.Equal(t, fmt.Sprintf("%x", sc.snap.SourceClusterID), em.Status.SourceClusterID) + assert.Equal(t, fmt.Sprintf("%x", sc.snap.TargetClusterID), em.Status.TargetClusterID) + assert.Equal(t, int64(1200), em.Status.LastAppliedRevision) + assert.Equal(t, pod.Name, em.Status.AgentPod) + assert.Equal(t, em.Generation, em.Status.ObservedGeneration) + require.Equal(t, 1, sc.callCount()) + assert.Equal(t, "10.1.2.3:8080", sc.calls[0]) +} + +func TestEtcdMirrorReconcile_ConditionFixtures(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + ns := createTestNamespace(t) + + t.Run("lagging sustained via the injected ledger", func(t *testing.T) { + sc := &fakeStatusClient{} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + snap := healthySnapshot() + snap.SourceRevision = snap.Watermark + lagRevisionThreshold + 100 + sc.set(snap, nil) + em := setupActiveMirror(t, r, ns, nil) + + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionReplicationLagExceeded, + metav1.ConditionFalse, reasonWithinSustainWindow) + + // Backdate the ledger past the sustain window. + r.mu.Lock() + r.lagSince[got.UID] = time.Now().Add(-lagSustainDuration - time.Minute) + r.mu.Unlock() + reconcileMirrorOnce(t, r, em) + got = refreshMirror(t, em) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionReplicationLagExceeded, + metav1.ConditionTrue, reasonLagSustained) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionInvariantsHeld, + metav1.ConditionFalse, reasonLagExceeded) + }) + + t.Run("forced resync reports InitialSync plus Compacted", func(t *testing.T) { + sc := &fakeStatusClient{snap: healthySnapshot()} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + em := setupActiveMirror(t, r, ns, nil) + reconcileMirrorOnce(t, r, em) // one healthy poll: InitialSyncComplete=True + + // The agent's fresh scan zeroes its process-local completion time — + // the state a real forced resync serves on the wire. + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseInitialSync + snap.Compacted = true + snap.ForcedResyncCount = 1 + snap.LastResyncReason = mirroragent.ResyncReasonCompacted + snap.InitialSyncCompletionTime = time.Time{} + sc.set(snap, nil) + + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhaseInitialSync, got.Status.Phase) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionCompacted, metav1.ConditionTrue, + ecv1alpha1.EtcdMirrorReasonForcedResync) + assert.Equal(t, int32(1), got.Status.ForcedResyncCount) + // Durable across forced resyncs (Compacted covers those). + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionInitialSyncComplete, metav1.ConditionTrue, reasonComplete) + }) + + t.Run("throttled while degraded", func(t *testing.T) { + sc := &fakeStatusClient{} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseDegraded + snap.Throttled = true + sc.set(snap, nil) + em := setupActiveMirror(t, r, ns, nil) + + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhaseDegraded, got.Status.Phase) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionTargetThrottled, metav1.ConditionTrue, "") + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, reasonBackoff) + }) + + t.Run("quota exhausted", func(t *testing.T) { + sc := &fakeStatusClient{} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + snap := healthySnapshot() + snap.QuotaExhausted = true + sc.set(snap, nil) + em := setupActiveMirror(t, r, ns, nil) + + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionTargetQuotaExhausted, metav1.ConditionTrue, "") + }) + + t.Run("resync loop", func(t *testing.T) { + sc := &fakeStatusClient{} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + snap := healthySnapshot() + snap.ResyncLoopDetected = true + sc.set(snap, nil) + em := setupActiveMirror(t, r, ns, nil) + + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionResyncLoopDetected, metav1.ConditionTrue, "") + }) + + t.Run("drift counts break InvariantsHeld", func(t *testing.T) { + sc := &fakeStatusClient{} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + snap := healthySnapshot() + snap.LastReconcileDrift = &mirroragent.Drift{MissingKeys: 4, OrphanKeys: 1} + sc.set(snap, nil) + em := setupActiveMirror(t, r, ns, nil) + + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + cond := requireCond(t, got, ecv1alpha1.EtcdMirrorConditionDriftDetected, metav1.ConditionTrue, "") + assert.Contains(t, cond.Message, "4 missing") + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionInvariantsHeld, metav1.ConditionFalse, reasonDriftDetected) + require.NotNil(t, got.Status.LastReconciliationDrift) + assert.Equal(t, int64(4), got.Status.LastReconciliationDrift.MissingKeys) + }) + + t.Run("drain maps Drained and populates cutover", func(t *testing.T) { + sc := &fakeStatusClient{} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseDrained + snap.CutoverReady = true + snap.Cutover = &mirroragent.CutoverStatus{ + DrainTargetRevision: 1234, + DrainedRevision: 1234, + VerifiedTime: time.Now(), + SourceKeyCount: 500, + TargetKeyCount: 500, + LeasedKeyCount: 2, + } + sc.set(snap, nil) + em := setupActiveMirror(t, r, ns, func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Mode = ecv1alpha1.EtcdMirrorModeDrain + }) + // Drain propagates through the args (spec-driven rollout). + dep := getAgentDeployment(t, em) + assert.Contains(t, dep.Spec.Template.Spec.Containers[0].Args, "--mode=Drain") + + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhaseSyncing, got.Status.Phase) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionCutoverReady, metav1.ConditionTrue, "") + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionTrue, reasonDrainComplete) + require.NotNil(t, got.Status.Cutover) + assert.Equal(t, int64(1234), got.Status.Cutover.DrainedRevision) + }) + + t.Run("failed with UnsupportedVersion", func(t *testing.T) { + sc := &fakeStatusClient{} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseFailed + snap.LastError = "source version 3.3.0 is below the 3.4 floor" + snap.LastErrorReason = "UnsupportedVersion" + sc.set(snap, nil) + em := setupActiveMirror(t, r, ns, nil) + + res := reconcileMirrorOnce(t, r, em) + assert.Equal(t, slowRequeueInterval, res.RequeueAfter, "Failed mirrors poll at the slow rate") + got := refreshMirror(t, em) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhaseFailed, got.Status.Phase) + cond := requireCond(t, got, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, + ecv1alpha1.EtcdMirrorReasonUnsupportedVersion) + assert.Equal(t, snap.LastError, cond.Message) + }) + + t.Run("failed with CheckpointInvalid", func(t *testing.T) { + sc := &fakeStatusClient{} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseFailed + snap.LastError = "checkpoint payload corrupt" + snap.LastErrorReason = "CheckpointInvalid" + sc.set(snap, nil) + em := setupActiveMirror(t, r, ns, nil) + + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, + ecv1alpha1.EtcdMirrorReasonCheckpointInvalid) + }) + + t.Run("learner endpoint", func(t *testing.T) { + sc := &fakeStatusClient{} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + snap := healthySnapshot() + snap.TargetLearner = true + sc.set(snap, nil) + em := setupActiveMirror(t, r, ns, nil) + + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + cond := requireCond(t, got, ecv1alpha1.EtcdMirrorConditionLearnerEndpoint, metav1.ConditionTrue, "") + assert.Contains(t, cond.Message, "target") + }) +} + +func TestEtcdMirrorReconcile_EmptyTargetViolation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{} + r, rec := newTestMirrorReconciler(sc, &fakeCleaner{}) + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseFailed + snap.LastError = "destination prefix /mirrored/ holds 12 keys" + snap.LastErrorReason = "EmptyTargetViolation" + sc.set(snap, nil) + ns := createTestNamespace(t) + em := setupActiveMirror(t, r, ns, nil) + + reconcileMirrorOnce(t, r, em) + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhaseFailed, got.Status.Phase) + cond := requireCond(t, got, ecv1alpha1.EtcdMirrorConditionEmptyTargetViolation, metav1.ConditionTrue, "") + assert.Contains(t, cond.Message, `etcdctl del "/mirrored/" "/mirrored0"`) + assert.Equal(t, 1, rec.countReason(ecv1alpha1.EtcdMirrorConditionEmptyTargetViolation), + "the Warning must fire exactly once across reconciles") + assert.Contains(t, rec.lastNote(ecv1alpha1.EtcdMirrorConditionEmptyTargetViolation), + `etcdctl del "/mirrored/" "/mirrored0"`) +} + +func TestEtcdMirrorReconcile_NoPodYet(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{snap: healthySnapshot()} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + ns := createTestNamespace(t) + em := newEnvtestMirror(t, ns, nil) + reconcileMirrorOnce(t, r, em) + reconcileMirrorOnce(t, r, em) + res := reconcileMirrorOnce(t, r, em) + assert.Equal(t, statusPollInterval, res.RequeueAfter) + got := refreshMirror(t, em) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, reasonAgentPodNotReady) + assert.Zero(t, sc.callCount(), "statusz must never be polled without a running pod IP") +} + +func TestEtcdMirrorReconcile_PodNotReady(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{snap: healthySnapshot()} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + ns := createTestNamespace(t) + em := newEnvtestMirror(t, ns, nil) + reconcileMirrorOnce(t, r, em) + reconcileMirrorOnce(t, r, em) + + // Pod exists but is not Running and has no IP. + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: deploymentNameForEtcdMirror(em) + "-", + Namespace: em.Namespace, + Labels: etcdMirrorAgentLabels(em), + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "mirror-agent", Image: testAgentImage}}}, + } + require.NoError(t, k8sClient.Create(t.Context(), pod)) + + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, reasonAgentPodNotReady) + assert.Equal(t, pod.Name, got.Status.AgentPod) + assert.Zero(t, sc.callCount()) +} + +func TestEtcdMirrorReconcile_StatuszTimeout(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{snap: healthySnapshot()} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + ns := createTestNamespace(t) + em := setupActiveMirror(t, r, ns, nil) + + // One healthy poll populates status. + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + require.Equal(t, ecv1alpha1.EtcdMirrorPhaseSyncing, got.Status.Phase) + syncTime := got.Status.LastStatusSyncTime + require.NotNil(t, syncTime) + + // Then the agent stops answering. + sc.set(nil, errors.New("dial tcp 10.1.2.3:8080: i/o timeout")) + res := reconcileMirrorOnce(t, r, em) + assert.Equal(t, statusPollInterval, res.RequeueAfter) + got = refreshMirror(t, em) + cond := requireCond(t, got, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionUnknown, + reasonAgentStatusUnreachable) + assert.Contains(t, cond.Message, "i/o timeout") + // ALL prior fields retained, including phase: Degraded is the agent's + // backoff state, which a controller-side poll failure says nothing about. + assert.Equal(t, ecv1alpha1.EtcdMirrorPhaseSyncing, got.Status.Phase) + assert.Equal(t, int64(1200), got.Status.LastAppliedRevision) + assert.True(t, got.Status.LastStatusSyncTime.Equal(syncTime), "LastStatusSyncTime must not advance") +} + +func TestEtcdMirrorReconcile_SnapshotDecodeFailure(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{snap: healthySnapshot()} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + ns := createTestNamespace(t) + em := setupActiveMirror(t, r, ns, nil) + + // A healthy poll first, so the retained phase is observable. + reconcileMirrorOnce(t, r, em) + sc.set(nil, &snapshotDecodeError{err: errors.New("invalid character '<'")}) + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhaseSyncing, got.Status.Phase, "poll failure must not fabricate Degraded") + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionUnknown, reasonSnapshotDecodeFailed) +} + +func TestEtcdMirrorReconcile_PrefixConflict(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{snap: healthySnapshot()} + r, rec := newTestMirrorReconciler(sc, &fakeCleaner{}) + ns := createTestNamespace(t) + sharedTarget := []string{"shared-prefix-tgt.example.com:2379"} + + older := newEnvtestMirror(t, ns, func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Target.EndpointList = sharedTarget + em.Spec.Target.Prefix = "/mirrored/" + }) + reconcileMirrorOnce(t, r, older) + reconcileMirrorOnce(t, r, older) + + // creationTimestamp has 1s resolution; make the second CR strictly newer. + time.Sleep(1100 * time.Millisecond) + newer := newEnvtestMirror(t, ns, func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Target.EndpointList = sharedTarget + em.Spec.Target.Prefix = "/mirrored/nested/" + }) + reconcileMirrorOnce(t, r, newer) + res := reconcileMirrorOnce(t, r, newer) + assert.Equal(t, slowRequeueInterval, res.RequeueAfter) + + got := refreshMirror(t, newer) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhasePending, got.Status.Phase) + cond := requireCond(t, got, ecv1alpha1.EtcdMirrorConditionPrefixConflict, metav1.ConditionTrue, reasonConflict) + assert.Contains(t, cond.Message, older.Name) + // The loser's OTHER guard condition is still evaluated (a stale True from + // a since-deleted reverse sibling must not survive the early block). + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionDirectionConflict, metav1.ConditionFalse, reasonNoConflict) + dep := getAgentDeployment(t, newer) + assert.Equal(t, int32(0), *dep.Spec.Replicas, "the loser's Deployment must be scaled to zero") + require.Equal(t, 1, rec.countReason(ecv1alpha1.EtcdMirrorConditionPrefixConflict)) + + // Second reconcile: no extra Warning (transition-only). + reconcileMirrorOnce(t, r, newer) + assert.Equal(t, 1, rec.countReason(ecv1alpha1.EtcdMirrorConditionPrefixConflict)) + + // The older CR keeps running; its False honestly names the parked loser + // instead of claiming nothing overlaps. + reconcileMirrorOnce(t, r, older) + gotOlder := refreshMirror(t, older) + condOlder := requireCond(t, gotOlder, ecv1alpha1.EtcdMirrorConditionPrefixConflict, + metav1.ConditionFalse, reasonConflictWinner) + assert.Contains(t, condOlder.Message, newer.Name) + assert.NotEqual(t, ecv1alpha1.EtcdMirrorPhaseFailed, gotOlder.Status.Phase) + + // Deleting the older clears the conflict on the next reconcile. + require.NoError(t, k8sClient.Delete(t.Context(), gotOlder)) + reconcileMirrorOnce(t, r, older) // finalize path releases it + err := k8sClient.Get(t.Context(), types.NamespacedName{Namespace: ns, Name: older.Name}, &ecv1alpha1.EtcdMirror{}) + require.True(t, apierrors.IsNotFound(err)) + + reconcileMirrorOnce(t, r, newer) + got = refreshMirror(t, newer) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionPrefixConflict, metav1.ConditionFalse, reasonNoConflict) +} + +func TestEtcdMirrorReconcile_DirectionConflict(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{snap: healthySnapshot()} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + ns := createTestNamespace(t) + + older := newEnvtestMirror(t, ns, nil) + reconcileMirrorOnce(t, r, older) + time.Sleep(1100 * time.Millisecond) + newer := newEnvtestMirror(t, ns, nil) + reconcileMirrorOnce(t, r, newer) + + // Seed inverted runtime cluster-ID pairs (respelled endpoints would fool + // a spec comparison; the runtime IDs cannot be fooled). + seed := func(em *ecv1alpha1.EtcdMirror, src, tgt string) { + got := refreshMirror(t, em) + got.Status.SourceClusterID = src + got.Status.TargetClusterID = tgt + require.NoError(t, k8sClient.Status().Update(t.Context(), got)) + } + seed(older, "aaaa1111", "bbbb2222") + seed(newer, "bbbb2222", "aaaa1111") + + res := reconcileMirrorOnce(t, r, newer) + assert.Equal(t, slowRequeueInterval, res.RequeueAfter) + gotNewer := refreshMirror(t, newer) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhasePending, gotNewer.Status.Phase) + requireCond(t, gotNewer, ecv1alpha1.EtcdMirrorConditionDirectionConflict, metav1.ConditionTrue, reasonConflict) + dep := getAgentDeployment(t, newer) + assert.Equal(t, int32(0), *dep.Spec.Replicas) + + // The winner reports the mutual condition too, but keeps running. + reconcileMirrorOnce(t, r, older) + gotOlder := refreshMirror(t, older) + requireCond(t, gotOlder, ecv1alpha1.EtcdMirrorConditionDirectionConflict, metav1.ConditionTrue, reasonConflict) + assert.NotEqual(t, ecv1alpha1.EtcdMirrorPhaseFailed, gotOlder.Status.Phase) + depOlder := getAgentDeployment(t, older) + assert.Equal(t, int32(1), *depOlder.Spec.Replicas) +} + +func TestEtcdMirrorReconcile_Paused(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{snap: healthySnapshot()} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + ns := createTestNamespace(t) + em := setupActiveMirror(t, r, ns, nil) + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + require.Equal(t, ecv1alpha1.EtcdMirrorPhaseSyncing, got.Status.Phase) + + got.Spec.Paused = true + require.NoError(t, k8sClient.Update(t.Context(), got)) + res := reconcileMirrorOnce(t, r, em) + assert.Equal(t, slowRequeueInterval, res.RequeueAfter) + got = refreshMirror(t, em) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhasePaused, got.Status.Phase) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, reasonPaused) + // Other status stays honest (retained, not zeroed). + assert.Equal(t, int64(1200), got.Status.LastAppliedRevision) + dep := getAgentDeployment(t, em) + assert.Equal(t, int32(0), *dep.Spec.Replicas) + + got.Spec.Paused = false + require.NoError(t, k8sClient.Update(t.Context(), got)) + reconcileMirrorOnce(t, r, em) + dep = getAgentDeployment(t, em) + assert.Equal(t, int32(1), *dep.Spec.Replicas) +} + +func TestEtcdMirrorReconcile_Finalizer(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + t.Run("deployment first, then checkpoint delete, retries on failure", func(t *testing.T) { + sc := &fakeStatusClient{snap: healthySnapshot()} + cleaner := &fakeCleaner{} + r, rec := newTestMirrorReconciler(sc, cleaner) + ns := createTestNamespace(t) + em := setupActiveMirror(t, r, ns, nil) + + require.NoError(t, k8sClient.Delete(t.Context(), refreshMirror(t, em))) + + // Pods still exist: the agent must stop before the key is deleted. + res := reconcileMirrorOnce(t, r, em) + assert.Equal(t, finalizerPodWait, res.RequeueAfter) + assert.Zero(t, cleaner.callCount()) + dep := &appsv1.Deployment{} + err := k8sClient.Get(t.Context(), + types.NamespacedName{Namespace: em.Namespace, Name: deploymentNameForEtcdMirror(em)}, dep) + if err == nil { + assert.NotNil(t, dep.DeletionTimestamp) + } else { + assert.True(t, apierrors.IsNotFound(err)) + } + + // Kill the pod (envtest has no kubelet to do it). + pods := &corev1.PodList{} + require.NoError(t, k8sClient.List(t.Context(), pods, + client.InNamespace(em.Namespace), client.MatchingLabels(etcdMirrorAgentLabels(em)))) + for i := range pods.Items { + require.NoError(t, k8sClient.Delete(t.Context(), &pods.Items[i], client.GracePeriodSeconds(0))) + } + + // Cleaner failure: finalizer retained, Warning, bounded retry. + cleaner.setErr(errors.New("target unreachable")) + res = reconcileMirrorOnce(t, r, em) + assert.Positive(t, res.RequeueAfter) + assert.Equal(t, 1, cleaner.callCount()) + assert.Equal(t, 1, rec.countReason("CheckpointCleanupFailed")) + got := refreshMirror(t, em) + assert.Contains(t, got.Finalizers, checkpointCleanupFinalizer) + + // Success: the checkpoint target carries the endpoints and the + // default reserved key; the finalizer is released. + cleaner.setErr(nil) + reconcileMirrorOnce(t, r, em) + require.Equal(t, 2, cleaner.callCount()) + tgt := cleaner.calls[1] + assert.Equal(t, got.Spec.Target.EndpointList, tgt.Endpoints) + assert.Nil(t, tgt.TLS) + assert.Equal(t, "/mirrored/"+mirroragent.DefaultCheckpointKeySuffix, tgt.Key) + err = k8sClient.Get(t.Context(), + types.NamespacedName{Namespace: em.Namespace, Name: em.Name}, &ecv1alpha1.EtcdMirror{}) + assert.True(t, apierrors.IsNotFound(err)) + }) + + t.Run("explicit checkpoint key is deleted verbatim", func(t *testing.T) { + sc := &fakeStatusClient{snap: healthySnapshot()} + cleaner := &fakeCleaner{} + r, _ := newTestMirrorReconciler(sc, cleaner) + ns := createTestNamespace(t) + em := newEnvtestMirror(t, ns, func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Checkpoint = &ecv1alpha1.EtcdMirrorCheckpointSpec{Key: "/mirrored/\x00custom-cp"} + }) + reconcileMirrorOnce(t, r, em) + reconcileMirrorOnce(t, r, em) // Deployment exists, no pods + require.NoError(t, k8sClient.Delete(t.Context(), refreshMirror(t, em))) + reconcileMirrorOnce(t, r, em) // deletes Deployment; no pods -> cleaner runs + require.Equal(t, 1, cleaner.callCount()) + assert.Equal(t, "/mirrored/\x00custom-cp", cleaner.calls[0].Key) + }) + + t.Run("foreign-owned fence is left in place and finalization completes", func(t *testing.T) { + sc := &fakeStatusClient{snap: healthySnapshot()} + cleaner := &fakeCleaner{skip: `the stored fence is owned by link "other-uid", not this mirror; leaving the key in place`} + r, rec := newTestMirrorReconciler(sc, cleaner) + ns := createTestNamespace(t) + em := newEnvtestMirror(t, ns, nil) + reconcileMirrorOnce(t, r, em) + reconcileMirrorOnce(t, r, em) // Deployment exists, no pods + require.NoError(t, k8sClient.Delete(t.Context(), refreshMirror(t, em))) + + reconcileMirrorOnce(t, r, em) + require.Equal(t, 1, cleaner.callCount()) + assert.Equal(t, string(em.UID), cleaner.calls[0].LinkUID, + "the cleaner must receive this CR's link UID for the ownership check") + assert.Equal(t, 1, rec.countReason("CheckpointNotOwned")) + assert.Contains(t, rec.lastNote("CheckpointNotOwned"), "other-uid") + err := k8sClient.Get(t.Context(), + types.NamespacedName{Namespace: em.Namespace, Name: em.Name}, &ecv1alpha1.EtcdMirror{}) + assert.True(t, apierrors.IsNotFound(err), "a foreign fence must not block deletion") + }) + + t.Run("skip annotation is the escape hatch", func(t *testing.T) { + sc := &fakeStatusClient{snap: healthySnapshot()} + cleaner := &fakeCleaner{} + r, rec := newTestMirrorReconciler(sc, cleaner) + ns := createTestNamespace(t) + em := newEnvtestMirror(t, ns, nil) + reconcileMirrorOnce(t, r, em) + + got := refreshMirror(t, em) + got.Annotations = map[string]string{skipCheckpointCleanupAnnotation: "true"} + require.NoError(t, k8sClient.Update(t.Context(), got)) + require.NoError(t, k8sClient.Delete(t.Context(), got)) + + reconcileMirrorOnce(t, r, em) + assert.Zero(t, cleaner.callCount()) + assert.Equal(t, 1, rec.countReason("CheckpointCleanupSkipped")) + // The key is %q-rendered in the note, so the NUL byte appears escaped. + assert.Contains(t, rec.lastNote("CheckpointCleanupSkipped"), "etcdmirror-checkpoint") + err := k8sClient.Get(t.Context(), + types.NamespacedName{Namespace: em.Namespace, Name: em.Name}, &ecv1alpha1.EtcdMirror{}) + assert.True(t, apierrors.IsNotFound(err)) + }) +} + +func TestEtcdMirrorReconcile_EventsEmittedOnce(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{} + r, rec := newTestMirrorReconciler(sc, &fakeCleaner{}) + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseInitialSync + snap.Compacted = true + snap.ForcedResyncCount = 1 + snap.LastResyncReason = mirroragent.ResyncReasonClusterIDMismatch + snap.ScanRestartCount = 2 + snap.LastScanRestartCause = mirroragent.ScanRestartWatchBufferOverflow + sc.set(snap, nil) + ns := createTestNamespace(t) + em := setupActiveMirror(t, r, ns, nil) + + reconcileMirrorOnce(t, r, em) + reconcileMirrorOnce(t, r, em) // identical snapshot: nothing new + assert.Equal(t, 1, rec.countReason(ecv1alpha1.EtcdMirrorEventForcedResyncStarted)) + assert.Equal(t, 1, rec.countReason(ecv1alpha1.EtcdMirrorEventCheckpointInvalidated)) + assert.Equal(t, 1, rec.countReason(ecv1alpha1.EtcdMirrorEventInitialSyncCompactionRaced)) + assert.Contains(t, rec.lastNote(ecv1alpha1.EtcdMirrorEventInitialSyncCompactionRaced), + string(mirroragent.ScanRestartWatchBufferOverflow)) + + // The resync completes: one Normal completion event. + done := *snap + done.Phase = mirroragent.PhaseSyncing + done.Compacted = false + sc.set(&done, nil) + reconcileMirrorOnce(t, r, em) + reconcileMirrorOnce(t, r, em) + assert.Equal(t, 1, rec.countReason(ecv1alpha1.EtcdMirrorEventForcedResyncCompleted)) + + // A second forced resync emits again. + again := done + again.Phase = mirroragent.PhaseInitialSync + again.Compacted = true + again.ForcedResyncCount = 2 + again.LastResyncReason = mirroragent.ResyncReasonCompacted + sc.set(&again, nil) + reconcileMirrorOnce(t, r, em) + assert.Equal(t, 2, rec.countReason(ecv1alpha1.EtcdMirrorEventForcedResyncStarted)) + assert.Equal(t, 1, rec.countReason(ecv1alpha1.EtcdMirrorEventCheckpointInvalidated), + "a Compacted-triggered resync is not a checkpoint invalidation") +} + +// selfSignedCertPEM builds a throwaway certificate expiring at notAfter. +func selfSignedCertPEM(t *testing.T, notAfter time.Time) ([]byte, []byte) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "etcdmirror-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: notAfter, + } + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key) + require.NoError(t, err) + keyDER, err := x509.MarshalECPrivateKey(key) + require.NoError(t, err) + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) +} + +func TestEtcdMirrorReconcile_CertExpiryWarning(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + ns := createTestNamespace(t) + + makeTLSSecret := func(name string, notAfter time.Time) { + certPEM, keyPEM := selfSignedCertPEM(t, notAfter) + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Data: map[string][]byte{"tls.crt": certPEM, "tls.key": keyPEM}, + } + require.NoError(t, k8sClient.Create(t.Context(), secret)) + } + + t.Run("expiring in 7d warns once", func(t *testing.T) { + sc := &fakeStatusClient{snap: healthySnapshot()} + r, rec := newTestMirrorReconciler(sc, &fakeCleaner{}) + makeTLSSecret("soon-tls", time.Now().Add(7*24*time.Hour)) + em := setupActiveMirror(t, r, ns, func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Source.TLS = &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: &corev1.LocalObjectReference{Name: "soon-tls"}, + } + }) + reconcileMirrorOnce(t, r, em) + reconcileMirrorOnce(t, r, em) // damped: still one + require.Equal(t, 1, rec.countReason(ecv1alpha1.EtcdMirrorEventCertificateExpiringSoon)) + note := rec.lastNote(ecv1alpha1.EtcdMirrorEventCertificateExpiringSoon) + assert.Contains(t, note, "source") + assert.Contains(t, note, "tls.crt") + }) + + t.Run("expiring in 60d does not warn", func(t *testing.T) { + sc := &fakeStatusClient{snap: healthySnapshot()} + r, rec := newTestMirrorReconciler(sc, &fakeCleaner{}) + makeTLSSecret("later-tls", time.Now().Add(60*24*time.Hour)) + em := setupActiveMirror(t, r, ns, func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Source.TLS = &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: &corev1.LocalObjectReference{Name: "later-tls"}, + } + }) + reconcileMirrorOnce(t, r, em) + assert.Zero(t, rec.countReason(ecv1alpha1.EtcdMirrorEventCertificateExpiringSoon)) + }) +} + +func TestEtcdMirrorReconcile_InsecureSkipVerifyWarning(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{snap: healthySnapshot()} + r, rec := newTestMirrorReconciler(sc, &fakeCleaner{}) + ns := createTestNamespace(t) + em := setupActiveMirror(t, r, ns, func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Target.TLS = &ecv1alpha1.EtcdMirrorTLS{ + InsecureSkipVerify: true, + InsecureSkipVerifyAcknowledgeRisk: true, + } + }) + reconcileMirrorOnce(t, r, em) + reconcileMirrorOnce(t, r, em) + require.Equal(t, 1, rec.countReason(ecv1alpha1.EtcdMirrorEventInsecureSkipVerifyEnabled)) + assert.Contains(t, rec.lastNote(ecv1alpha1.EtcdMirrorEventInsecureSkipVerifyEnabled), "target") + + // The standing warning has no reachability qualifier: a mirror that + // never passes validation (here: no agent image) must still warn. + rBlocked, recBlocked := newTestMirrorReconciler(sc, &fakeCleaner{}) + rBlocked.AgentImage = "" + emBlocked := newEnvtestMirror(t, ns, func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Source.TLS = &ecv1alpha1.EtcdMirrorTLS{ + InsecureSkipVerify: true, + InsecureSkipVerifyAcknowledgeRisk: true, + } + }) + reconcileMirrorOnce(t, rBlocked, emBlocked) + reconcileMirrorOnce(t, rBlocked, emBlocked) + require.Equal(t, 1, recBlocked.countReason(ecv1alpha1.EtcdMirrorEventInsecureSkipVerifyEnabled)) + assert.Contains(t, recBlocked.lastNote(ecv1alpha1.EtcdMirrorEventInsecureSkipVerifyEnabled), "source") +} + +func TestEtcdMirrorReconcile_ServiceRefPortResolution(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + ns := createTestNamespace(t) + + t.Run("named port resolves to a numeric dial port", func(t *testing.T) { + sc := &fakeStatusClient{snap: healthySnapshot()} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "etcd-target-client", Namespace: ns}, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Name: "client", Port: 2379}}, + }, + } + require.NoError(t, k8sClient.Create(t.Context(), svc)) + + em := newEnvtestMirror(t, ns, func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Target.ServiceRef = &ecv1alpha1.EtcdMirrorServiceRef{Name: "etcd-target-client"} + }) + reconcileMirrorOnce(t, r, em) + reconcileMirrorOnce(t, r, em) + dep := getAgentDeployment(t, em) + assert.Contains(t, dep.Spec.Template.Spec.Containers[0].Args, + "--target-endpoints=etcd-target-client."+ns+".svc.cluster.local:2379") + }) + + t.Run("missing Service parks the CR in Pending", func(t *testing.T) { + sc := &fakeStatusClient{snap: healthySnapshot()} + r, rec := newTestMirrorReconciler(sc, &fakeCleaner{}) + em := newEnvtestMirror(t, ns, func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Target.ServiceRef = &ecv1alpha1.EtcdMirrorServiceRef{Name: "absent-service"} + }) + reconcileMirrorOnce(t, r, em) + res := reconcileMirrorOnce(t, r, em) + assert.Positive(t, res.RequeueAfter) + got := refreshMirror(t, em) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhasePending, got.Status.Phase) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, reasonServiceNotFound) + assert.Equal(t, 1, rec.countReason(reasonServiceNotFound)) + }) +} + +// TestEtcdMirrorReconcile_CountersSurvivePodRestart pins the "Monotonic, +// never reset" status contract: the agent's counters are process memory, so a +// pod restart reports 0 and the controller must rebase, not regress. +func TestEtcdMirrorReconcile_CountersSurvivePodRestart(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{} + r, rec := newTestMirrorReconciler(sc, &fakeCleaner{}) + snap := healthySnapshot() + snap.ForcedResyncCount = 2 + snap.LastResyncReason = mirroragent.ResyncReasonCompacted + snap.ScanRestartCount = 3 + snap.LastScanRestartCause = mirroragent.ScanRestartWatchBufferOverflow + sc.set(snap, nil) + ns := createTestNamespace(t) + em := setupActiveMirror(t, r, ns, nil) + + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + require.Equal(t, int32(2), got.Status.ForcedResyncCount) + require.Equal(t, int64(3), got.Status.ScanRestartCount) + + // The pod restarts: fresh process, counters back to zero on the wire. + pods := &corev1.PodList{} + require.NoError(t, k8sClient.List(t.Context(), pods, + client.InNamespace(em.Namespace), client.MatchingLabels(etcdMirrorAgentLabels(em)))) + for i := range pods.Items { + require.NoError(t, k8sClient.Delete(t.Context(), &pods.Items[i], client.GracePeriodSeconds(0))) + } + makeAgentPodReady(t, em) + restarted := *snap + restarted.ForcedResyncCount = 0 + restarted.ScanRestartCount = 0 + sc.set(&restarted, nil) + + events := rec.countReason(ecv1alpha1.EtcdMirrorEventForcedResyncStarted) + reconcileMirrorOnce(t, r, em) + got = refreshMirror(t, em) + assert.Equal(t, int32(2), got.Status.ForcedResyncCount, "a pod restart must never regress the counter") + assert.Equal(t, int64(3), got.Status.ScanRestartCount) + assert.Equal(t, events, rec.countReason(ecv1alpha1.EtcdMirrorEventForcedResyncStarted), + "a rebased counter must not re-fire events") + + // A real resync in the new process still advances and still fires. + resynced := restarted + resynced.ForcedResyncCount = 1 + sc.set(&resynced, nil) + reconcileMirrorOnce(t, r, em) + got = refreshMirror(t, em) + assert.Equal(t, int32(3), got.Status.ForcedResyncCount) + assert.Equal(t, events+1, rec.countReason(ecv1alpha1.EtcdMirrorEventForcedResyncStarted)) +} + +// TestEtcdMirrorReconcile_ValidationAfterWorkloadExists: deleting a +// referenced Secret must not claim Pending ("workload not created yet") for a +// mirror whose agent keeps running on mounted material. +func TestEtcdMirrorReconcile_ValidationAfterWorkloadExists(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{snap: healthySnapshot()} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + ns := createTestNamespace(t) + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "vanishing-auth", Namespace: ns}, + Data: map[string][]byte{"username": []byte("u"), "password": []byte("p")}, + } + require.NoError(t, k8sClient.Create(t.Context(), secret)) + em := setupActiveMirror(t, r, ns, func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Source.Auth = &ecv1alpha1.EtcdMirrorAuth{ + SecretRef: corev1.LocalObjectReference{Name: "vanishing-auth"}, + } + }) + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + require.Equal(t, ecv1alpha1.EtcdMirrorPhaseSyncing, got.Status.Phase) + + require.NoError(t, k8sClient.Delete(t.Context(), secret)) + reconcileMirrorOnce(t, r, em) + got = refreshMirror(t, em) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhaseSyncing, got.Status.Phase, + "an existing workload's phase must be retained, not overwritten with Pending") + cond := requireCond(t, got, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, reasonSecretNotFound) + assert.Contains(t, cond.Message, "keeps running") +} + +func TestEtcdMirrorReconcile_AgentImageNotConfigured(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{snap: healthySnapshot()} + r, rec := newTestMirrorReconciler(sc, &fakeCleaner{}) + r.AgentImage = "" + ns := createTestNamespace(t) + em := newEnvtestMirror(t, ns, nil) + reconcileMirrorOnce(t, r, em) + res := reconcileMirrorOnce(t, r, em) + assert.Positive(t, res.RequeueAfter) + + got := refreshMirror(t, em) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhasePending, got.Status.Phase) + requireCond(t, got, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, reasonAgentImageNotConfigured) + assert.Equal(t, 1, rec.countReason(reasonAgentImageNotConfigured)) + + err := k8sClient.Get(t.Context(), + types.NamespacedName{Namespace: ns, Name: deploymentNameForEtcdMirror(em)}, &appsv1.Deployment{}) + assert.True(t, apierrors.IsNotFound(err), "no Deployment may be created without an agent image") +} + +func TestEtcdMirrorReconcile_MetricsLifecycle(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + sc := &fakeStatusClient{snap: healthySnapshot()} + r, _ := newTestMirrorReconciler(sc, &fakeCleaner{}) + ns := createTestNamespace(t) + em := setupActiveMirror(t, r, ns, nil) + + // A failed /statusz poll must still update the gauges: the controller-side + // series are the alerting surface when the agent is absent or unreachable. + sc.set(nil, errors.New("dial tcp 10.1.2.3:8080: i/o timeout")) + reconcileMirrorOnce(t, r, em) + got := refreshMirror(t, em) + assert.Equal(t, 1.0, conditionValue(got, ecv1alpha1.EtcdMirrorConditionAvailable, "unknown")) + assert.Zero(t, conditionValue(got, ecv1alpha1.EtcdMirrorConditionAvailable, "true")) + assert.Equal(t, 1.0, phaseValue(got, got.Status.Phase)) + + // Delete the CR: series must vanish on the FIRST finalize reconcile (the + // pod-wait requeue) — checkpoint cleanup can retry unboundedly and + // frozen pre-delete gauges would misreport for that whole window. + require.NoError(t, k8sClient.Delete(t.Context(), got)) + reconcileMirrorOnce(t, r, em) // deletes the Deployment, waits for pods + assert.Zero(t, mirrorSeriesCount(t, "etcd_mirror_phase", em.Namespace, em.Name)) + assert.Zero(t, mirrorSeriesCount(t, "etcd_mirror_condition", em.Namespace, em.Name)) + pods := &corev1.PodList{} + require.NoError(t, k8sClient.List(t.Context(), pods, + client.InNamespace(em.Namespace), client.MatchingLabels(etcdMirrorAgentLabels(em)))) + for i := range pods.Items { + require.NoError(t, k8sClient.Delete(t.Context(), &pods.Items[i], client.GracePeriodSeconds(0))) + } + reconcileMirrorOnce(t, r, em) // checkpoint cleanup + finalizer removal + assert.Zero(t, mirrorSeriesCount(t, "etcd_mirror_phase", em.Namespace, em.Name)) + assert.Zero(t, mirrorSeriesCount(t, "etcd_mirror_condition", em.Namespace, em.Name)) +} diff --git a/internal/controller/etcdmirror_fakes_test.go b/internal/controller/etcdmirror_fakes_test.go new file mode 100644 index 00000000..ff3e19fc --- /dev/null +++ b/internal/controller/etcdmirror_fakes_test.go @@ -0,0 +1,294 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + "go.etcd.io/etcd-operator/pkg/mirroragent" +) + +// fakeStatusClient is the AgentStatusClient seam for envtest: agent pods +// never run there, so /statusz answers come from this fixture instead. +type fakeStatusClient struct { + mu sync.Mutex + snap *mirroragent.Snapshot + err error + calls []string +} + +func (f *fakeStatusClient) Snapshot(_ context.Context, addr string) (*mirroragent.Snapshot, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, addr) + if f.err != nil { + return nil, f.err + } + snap := *f.snap + return &snap, nil +} + +func (f *fakeStatusClient) set(snap *mirroragent.Snapshot, err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.snap, f.err = snap, err +} + +func (f *fakeStatusClient) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.calls) +} + +// fakeCleaner is the CheckpointCleaner seam (no reachable target etcd in +// envtest). +type fakeCleaner struct { + mu sync.Mutex + err error + skip string + calls []CheckpointTarget +} + +func (f *fakeCleaner) DeleteCheckpoint(_ context.Context, tgt CheckpointTarget) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, tgt) + return f.skip, f.err +} + +func (f *fakeCleaner) setErr(err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.err = err +} + +func (f *fakeCleaner) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.calls) +} + +type recordedEvent struct { + Type string + Reason string + Note string +} + +// fakeRecorder captures events behind the events.EventRecorder interface. +type fakeRecorder struct { + mu sync.Mutex + events []recordedEvent +} + +func (f *fakeRecorder) Eventf( + _ runtime.Object, _ runtime.Object, eventtype, reason, _, note string, args ...interface{}, +) { + f.mu.Lock() + defer f.mu.Unlock() + f.events = append(f.events, recordedEvent{Type: eventtype, Reason: reason, Note: fmt.Sprintf(note, args...)}) +} + +func (f *fakeRecorder) countReason(reason string) int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for _, e := range f.events { + if e.Reason == reason { + n++ + } + } + return n +} + +func (f *fakeRecorder) lastNote(reason string) string { + f.mu.Lock() + defer f.mu.Unlock() + for i := len(f.events) - 1; i >= 0; i-- { + if f.events[i].Reason == reason { + return f.events[i].Note + } + } + return "" +} + +const testAgentImage = "example.com/etcd-operator:test" + +func newTestMirrorReconciler(sc AgentStatusClient, cleaner CheckpointCleaner) (*EtcdMirrorReconciler, *fakeRecorder) { + rec := &fakeRecorder{} + return &EtcdMirrorReconciler{ + Client: k8sClient, + Scheme: scheme.Scheme, + Recorder: rec, + AgentImage: testAgentImage, + StatusClient: sc, + Cleaner: cleaner, + }, rec +} + +func createTestNamespace(t *testing.T) string { + t.Helper() + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "etcdmirror-test-"}} + require.NoError(t, k8sClient.Create(t.Context(), ns)) + return ns.Name +} + +// mirrorSeq makes every test mirror's default endpoint hosts unique, so the +// cluster-wide guard checks never cross-talk between tests. +var mirrorSeq atomic.Int64 + +// newEnvtestMirror creates (via the API server, so defaults and CEL apply) a +// minimal valid EtcdMirror with unique default endpoint hosts. +func newEnvtestMirror(t *testing.T, ns string, mutate func(*ecv1alpha1.EtcdMirror)) *ecv1alpha1.EtcdMirror { + t.Helper() + em := &ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "mirror-", Namespace: ns}, + Spec: ecv1alpha1.EtcdMirrorSpec{ + Source: ecv1alpha1.EtcdMirrorEndpoint{Prefix: "/registry/"}, + Target: ecv1alpha1.EtcdMirrorEndpoint{Prefix: "/mirrored/"}, + }, + } + if mutate != nil { + mutate(em) + } + // Default unique endpoints unless the mutation chose its own resolution. + n := mirrorSeq.Add(1) + if len(em.Spec.Source.EndpointList) == 0 && em.Spec.Source.ServiceRef == nil { + em.Spec.Source.EndpointList = []string{fmt.Sprintf("src-%d.example.com:2379", n)} + } + if len(em.Spec.Target.EndpointList) == 0 && em.Spec.Target.ServiceRef == nil { + em.Spec.Target.EndpointList = []string{fmt.Sprintf("tgt-%d.example.com:2379", n)} + } + require.NoError(t, k8sClient.Create(t.Context(), em)) + return em +} + +func mirrorRequest(em *ecv1alpha1.EtcdMirror) ctrl.Request { + return ctrl.Request{NamespacedName: types.NamespacedName{Namespace: em.Namespace, Name: em.Name}} +} + +func reconcileMirrorOnce(t *testing.T, r *EtcdMirrorReconciler, em *ecv1alpha1.EtcdMirror) ctrl.Result { + t.Helper() + res, err := r.Reconcile(t.Context(), mirrorRequest(em)) + require.NoError(t, err) + return res +} + +func refreshMirror(t *testing.T, em *ecv1alpha1.EtcdMirror) *ecv1alpha1.EtcdMirror { + t.Helper() + out := &ecv1alpha1.EtcdMirror{} + require.NoError(t, k8sClient.Get(t.Context(), types.NamespacedName{Namespace: em.Namespace, Name: em.Name}, out)) + return out +} + +// makeAgentPodReady creates a Running agent pod with a PodIP for em (envtest +// has no kubelet, so the test IS the kubelet). +func makeAgentPodReady(t *testing.T, em *ecv1alpha1.EtcdMirror) *corev1.Pod { + t.Helper() + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: deploymentNameForEtcdMirror(em) + "-", + Namespace: em.Namespace, + Labels: etcdMirrorAgentLabels(em), + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "mirror-agent", Image: testAgentImage}}, + }, + } + require.NoError(t, k8sClient.Create(t.Context(), pod)) + pod.Status.Phase = corev1.PodRunning + pod.Status.PodIP = "10.1.2.3" + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + require.NoError(t, k8sClient.Status().Update(t.Context(), pod)) + return pod +} + +// setupActiveMirror runs a CR through finalizer-add, Deployment creation, and +// pod readiness so the next reconcile polls the (fake) agent. +func setupActiveMirror( + t *testing.T, r *EtcdMirrorReconciler, ns string, mutate func(*ecv1alpha1.EtcdMirror), +) *ecv1alpha1.EtcdMirror { + t.Helper() + em := newEnvtestMirror(t, ns, mutate) + reconcileMirrorOnce(t, r, em) // adds finalizer + reconcileMirrorOnce(t, r, em) // creates Deployment + makeAgentPodReady(t, em) + return em +} + +// healthySnapshot is a steady-state Syncing fixture with a fresh +// verification pass. Cluster IDs are unique per call: the runtime IDs land in +// status and the guards compare them cluster-wide, so shared fixture IDs +// would (correctly!) raise PrefixConflict between unrelated tests' mirrors. +func healthySnapshot() *mirroragent.Snapshot { + now := time.Now() + n := uint64(mirrorSeq.Add(1)) //nolint:gosec // test counter, never negative + return &mirroragent.Snapshot{ + Phase: mirroragent.PhaseSyncing, + SourceVersion: "3.5.21", + TargetVersion: "3.6.10", + SourceClusterID: 0xa000000000000000 + n, + TargetClusterID: 0xb000000000000000 + n, + Watermark: 1200, + SourceRevision: 1234, + LastProgressTime: now, + InitialSyncKeyCount: 500, + InitialSyncTotalKeyCount: 500, + InitialSyncStartTime: now.Add(-2 * time.Hour), + InitialSyncCompletionTime: now.Add(-time.Hour), + LeaseBackedKeyCount: 3, + SourceKeyCount: 500, + TargetKeyCount: 500, + LastReconcileTime: now.Add(-10 * time.Minute), + LastReconcileDrift: &mirroragent.Drift{}, + } +} + +func findCond(em *ecv1alpha1.EtcdMirror, condType string) *metav1.Condition { + for i := range em.Status.Conditions { + if em.Status.Conditions[i].Type == condType { + return &em.Status.Conditions[i] + } + } + return nil +} + +func requireCond( + t *testing.T, em *ecv1alpha1.EtcdMirror, condType string, status metav1.ConditionStatus, reason string, +) *metav1.Condition { + t.Helper() + cond := findCond(em, condType) + require.NotNil(t, cond, "condition %s not present", condType) + require.Equal(t, status, cond.Status, "condition %s status (reason %s: %s)", condType, cond.Reason, cond.Message) + if reason != "" { + require.Equal(t, reason, cond.Reason, "condition %s reason", condType) + } + return cond +} diff --git a/internal/controller/etcdmirror_guards.go b/internal/controller/etcdmirror_guards.go new file mode 100644 index 00000000..939da616 --- /dev/null +++ b/internal/controller/etcdmirror_guards.go @@ -0,0 +1,206 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "fmt" + "strings" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +// effectiveDestPrefix is the range mirrored keys land under on the target +// (the first two terms of the rewrite formula). +func effectiveDestPrefix(em *ecv1alpha1.EtcdMirror) string { + return em.Spec.Target.Prefix + em.Spec.Sync.DestPrefix +} + +// prefixRangesOverlap reports whether the key ranges covered by two prefixes +// intersect. Prefix ranges are nested-or-disjoint, and the empty prefix is +// the whole keyspace, so prefix containment IS the overlap computation. +func prefixRangesOverlap(a, b string) bool { + return a == "" || b == "" || strings.HasPrefix(a, b) || strings.HasPrefix(b, a) +} + +// endpointIdentity is one CR side's best-available cluster identity: the +// runtime cluster ID when the agent has probed it, and the spec-derived +// identity (serviceRef coordinates or normalized endpoint set) before then. +type endpointIdentity struct { + // clusterID is the hex-formatted runtime cluster ID from status + // ("" = not yet probed). + clusterID string + // serviceKey is "ns/name:port" when the side uses a serviceRef. + serviceKey string + // endpoints is the normalized endpointList (scheme stripped, host + // case-folded, trailing slash trimmed). + endpoints map[string]struct{} +} + +func normalizeEndpoint(ep string) string { + ep = strings.TrimPrefix(ep, "https://") + ep = strings.TrimPrefix(ep, "http://") + ep = strings.TrimSuffix(ep, "/") + return strings.ToLower(strings.TrimSpace(ep)) +} + +// endpointIdentityFor builds the identity of one side of em. clusterID is the +// corresponding status field (source or target). +func endpointIdentityFor(em *ecv1alpha1.EtcdMirror, ep ecv1alpha1.EtcdMirrorEndpoint, clusterID string) endpointIdentity { + id := endpointIdentity{clusterID: clusterID} + if ep.ServiceRef != nil { + ns := ep.ServiceRef.Namespace + if ns == "" { + ns = em.Namespace + } + port := ep.ServiceRef.Port + if port == "" { + port = defaultClientPortName + } + id.serviceKey = ns + "/" + ep.ServiceRef.Name + ":" + port + return id + } + id.endpoints = make(map[string]struct{}, len(ep.EndpointList)) + for _, e := range ep.EndpointList { + if n := normalizeEndpoint(e); n != "" { + id.endpoints[n] = struct{}{} + } + } + return id +} + +// sameCluster decides whether two endpoint identities point at the same etcd +// cluster. When both runtime cluster IDs are known, ID equality decides alone +// — it is authoritative, survives respelled endpoints, and avoids false +// positives after an NLB DNS rotation. Otherwise fall back to the spec +// identity: same serviceRef coordinates, or a nonempty normalized-endpoint +// intersection. +func sameCluster(a, b endpointIdentity) bool { + if a.clusterID != "" && b.clusterID != "" { + return a.clusterID == b.clusterID + } + if a.serviceKey != "" && a.serviceKey == b.serviceKey { + return true + } + for e := range a.endpoints { + if _, ok := b.endpoints[e]; ok { + return true + } + } + return false +} + +// mirrorConflict names a guard hit: the condition to raise and a message +// naming the sibling and the colliding ranges/IDs. +type mirrorConflict struct { + conditionType string + sibling *ecv1alpha1.EtcdMirror + message string +} + +func targetIdentity(em *ecv1alpha1.EtcdMirror) endpointIdentity { + return endpointIdentityFor(em, em.Spec.Target, em.Status.TargetClusterID) +} + +func sourceIdentity(em *ecv1alpha1.EtcdMirror) endpointIdentity { + return endpointIdentityFor(em, em.Spec.Source, em.Status.SourceClusterID) +} + +// findPrefixConflict returns the first other EtcdMirror (any namespace) whose +// effective destination range overlaps em's on the same target cluster. +func findPrefixConflict(em *ecv1alpha1.EtcdMirror, all []ecv1alpha1.EtcdMirror) *mirrorConflict { + emTarget := targetIdentity(em) + emRange := effectiveDestPrefix(em) + for i := range all { + other := &all[i] + if other.UID == em.UID { + continue + } + if !sameCluster(emTarget, targetIdentity(other)) { + continue + } + otherRange := effectiveDestPrefix(other) + if !prefixRangesOverlap(emRange, otherRange) { + continue + } + return &mirrorConflict{ + conditionType: ecv1alpha1.EtcdMirrorConditionPrefixConflict, + sibling: other, + message: fmt.Sprintf( + "EtcdMirror %s/%s targets the overlapping effective destination range %q (this mirror: %q) on the same target cluster", + other.Namespace, other.Name, otherRange, emRange), + } + } + return nil +} + +// selfMirror reports whether em's source and target resolve to the same +// cluster (an intra-cluster prefix copy). Cluster-level inversion is +// trivially true between any two such mirrors on one cluster — all four +// identities equal — yet no two-way loop exists, so the direction guard +// excludes them. (An intra-cluster loop through overlapping prefixes needs +// prefix awareness the cluster-level check cannot express; the agent-side +// fence-ownership backstop still catches destructive overlaps.) +func selfMirror(em *ecv1alpha1.EtcdMirror) bool { + return sameCluster(sourceIdentity(em), targetIdentity(em)) +} + +// findDirectionConflict returns the first other EtcdMirror forming a two-way +// loop with em: runtime check first (both bound cluster-ID pairs known and +// exact inverses — catches respelled endpoints), then the spec-identity +// pre-runtime check (source/target identities crosswise equal). +func findDirectionConflict(em *ecv1alpha1.EtcdMirror, all []ecv1alpha1.EtcdMirror) *mirrorConflict { + if selfMirror(em) { + return nil + } + emSource, emTarget := sourceIdentity(em), targetIdentity(em) + for i := range all { + other := &all[i] + if other.UID == em.UID || selfMirror(other) { + continue + } + runtimeInverse := em.Status.SourceClusterID != "" && em.Status.TargetClusterID != "" && + other.Status.SourceClusterID == em.Status.TargetClusterID && + other.Status.TargetClusterID == em.Status.SourceClusterID + specInverse := sameCluster(emSource, targetIdentity(other)) && + sameCluster(emTarget, sourceIdentity(other)) + if !runtimeInverse && !specInverse { + continue + } + return &mirrorConflict{ + conditionType: ecv1alpha1.EtcdMirrorConditionDirectionConflict, + sibling: other, + message: fmt.Sprintf( + "EtcdMirror %s/%s mirrors the opposite direction (its source/target cluster IDs %q/%q are the inverse of this mirror's %q/%q) — two CRs forming a two-way loop", + other.Namespace, other.Name, + other.Status.SourceClusterID, other.Status.TargetClusterID, + em.Status.SourceClusterID, em.Status.TargetClusterID), + } + } + return nil +} + +// isConflictLoser picks the CR that stops on a conflict: the newer one by +// creationTimestamp, tiebroken by the greater UID string. The conflict is +// re-evaluated every reconcile, so deleting the sibling clears it; the +// agent-side PrefixConflictError (permanent) is the backstop for anything the +// spec comparison misses. +func isConflictLoser(em, other *ecv1alpha1.EtcdMirror) bool { + if em.CreationTimestamp.Equal(&other.CreationTimestamp) { + return string(em.UID) > string(other.UID) + } + return other.CreationTimestamp.Before(&em.CreationTimestamp) +} diff --git a/internal/controller/etcdmirror_guards_test.go b/internal/controller/etcdmirror_guards_test.go new file mode 100644 index 00000000..f162ca95 --- /dev/null +++ b/internal/controller/etcdmirror_guards_test.go @@ -0,0 +1,183 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +func TestEffectiveDestRangeOverlap(t *testing.T) { + cases := []struct { + name string + a, b string + want bool + }{ + {"nested", "/mirrored/", "/mirrored/sub/", true}, + {"nested reversed", "/mirrored/sub/", "/mirrored/", true}, + {"equal", "/mirrored/", "/mirrored/", true}, + {"disjoint", "/a/", "/b/", false}, + {"shared string prefix but disjoint ranges", "/mirrored-a/", "/mirrored-b/", false}, + {"one empty is whole keyspace", "", "/anything/", true}, + {"both empty", "", "", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, prefixRangesOverlap(tc.a, tc.b)) + assert.Equal(t, tc.want, prefixRangesOverlap(tc.b, tc.a)) + }) + } +} + +func guardMirror(ns string, target ecv1alpha1.EtcdMirrorEndpoint, targetClusterID string) *ecv1alpha1.EtcdMirror { + return &ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{Name: "g", Namespace: ns}, + Spec: ecv1alpha1.EtcdMirrorSpec{Target: target}, + Status: ecv1alpha1.EtcdMirrorStatus{TargetClusterID: targetClusterID}, + } +} + +func TestSameTargetClusterHeuristic(t *testing.T) { + epList := func(eps ...string) ecv1alpha1.EtcdMirrorEndpoint { + return ecv1alpha1.EtcdMirrorEndpoint{EndpointList: eps} + } + svcRef := func(name, ns, port string) ecv1alpha1.EtcdMirrorEndpoint { + return ecv1alpha1.EtcdMirrorEndpoint{ + ServiceRef: &ecv1alpha1.EtcdMirrorServiceRef{Name: name, Namespace: ns, Port: port}, + } + } + + cases := []struct { + name string + a, b *ecv1alpha1.EtcdMirror + want bool + }{ + { + name: "matching runtime IDs decide alone", + a: guardMirror("ns1", epList("a.example.com:2379"), "abc123"), + b: guardMirror("ns2", epList("completely-respelled.example.com:2379"), "abc123"), + want: true, + }, + { + name: "differing runtime IDs override an endpoint match", + a: guardMirror("ns1", epList("same.example.com:2379"), "abc123"), + b: guardMirror("ns2", epList("same.example.com:2379"), "def456"), + want: false, + }, + { + name: "endpoint normalization: scheme, case, trailing slash", + a: guardMirror("ns1", epList("https://ETCD.Example.com:2379/"), ""), + b: guardMirror("ns2", epList("etcd.example.com:2379"), ""), + want: true, + }, + { + name: "nonempty endpoint intersection suffices", + a: guardMirror("ns1", epList("a:2379", "shared:2379"), ""), + b: guardMirror("ns2", epList("shared:2379", "b:2379"), ""), + want: true, + }, + { + name: "disjoint endpoints", + a: guardMirror("ns1", epList("a:2379"), ""), + b: guardMirror("ns2", epList("b:2379"), ""), + want: false, + }, + { + name: "serviceRef namespace defaults to the CR namespace", + a: guardMirror("shared-ns", svcRef("etcd-client", "", "client"), ""), + b: guardMirror("other-ns", svcRef("etcd-client", "shared-ns", ""), ""), + want: true, + }, + { + name: "same service name in different namespaces is different", + a: guardMirror("ns1", svcRef("etcd-client", "", ""), ""), + b: guardMirror("ns2", svcRef("etcd-client", "", ""), ""), + want: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := sameCluster(targetIdentity(tc.a), targetIdentity(tc.b)) + assert.Equal(t, tc.want, got) + }) + } +} + +// TestDirectionConflictSkipsIntraClusterMirrors: two disjoint-prefix A->A +// copies satisfy cluster-level inversion trivially (all four IDs equal) but +// form no loop; the guard must not park either. +func TestDirectionConflictSkipsIntraClusterMirrors(t *testing.T) { + intra := func(name, uid, srcPrefix, dstPrefix string) ecv1alpha1.EtcdMirror { + return ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "ns", UID: types.UID(uid)}, + Spec: ecv1alpha1.EtcdMirrorSpec{ + Source: ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: []string{"etcd-a:2379"}, Prefix: srcPrefix, + }, + Target: ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: []string{"etcd-a:2379"}, Prefix: dstPrefix, + }, + }, + Status: ecv1alpha1.EtcdMirrorStatus{SourceClusterID: "aaaa", TargetClusterID: "aaaa"}, + } + } + all := []ecv1alpha1.EtcdMirror{ + intra("copy1", "uid-1", "/x/", "/y/"), + intra("copy2", "uid-2", "/p/", "/q/"), + } + assert.Nil(t, findDirectionConflict(&all[0], all)) + assert.Nil(t, findDirectionConflict(&all[1], all)) + + // A genuine cross-cluster inverse pair still trips the guard. + fwd := ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{Name: "fwd", Namespace: "ns", UID: "uid-f"}, + Status: ecv1alpha1.EtcdMirrorStatus{SourceClusterID: "aaaa", TargetClusterID: "bbbb"}, + } + rev := ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{Name: "rev", Namespace: "ns", UID: "uid-r"}, + Status: ecv1alpha1.EtcdMirrorStatus{SourceClusterID: "bbbb", TargetClusterID: "aaaa"}, + } + pair := []ecv1alpha1.EtcdMirror{fwd, rev} + require.NotNil(t, findDirectionConflict(&pair[0], pair)) +} + +func TestSameTargetClusterConflictLoser(t *testing.T) { + older := &ecv1alpha1.EtcdMirror{ObjectMeta: metav1.ObjectMeta{ + UID: "b-uid", CreationTimestamp: metav1.Unix(100, 0), + }} + newer := &ecv1alpha1.EtcdMirror{ObjectMeta: metav1.ObjectMeta{ + UID: "a-uid", CreationTimestamp: metav1.Unix(200, 0), + }} + assert.True(t, isConflictLoser(newer, older)) + assert.False(t, isConflictLoser(older, newer)) + + // equal timestamps: greater UID string loses + twinA := &ecv1alpha1.EtcdMirror{ObjectMeta: metav1.ObjectMeta{ + UID: "aaa", CreationTimestamp: metav1.Unix(100, 0), + }} + twinB := &ecv1alpha1.EtcdMirror{ObjectMeta: metav1.ObjectMeta{ + UID: "bbb", CreationTimestamp: metav1.Unix(100, 0), + }} + assert.True(t, isConflictLoser(twinB, twinA)) + assert.False(t, isConflictLoser(twinA, twinB)) +} diff --git a/internal/controller/etcdmirror_metrics.go b/internal/controller/etcdmirror_metrics.go new file mode 100644 index 00000000..596f5c5a --- /dev/null +++ b/internal/controller/etcdmirror_metrics.go @@ -0,0 +1,105 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "strings" + + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +const ( + metricLabelNamespace = "namespace" + metricLabelName = "name" +) + +// Controller-side gauges mirroring EtcdMirror status. They exist so an agent +// that never scheduled (image typo, unschedulable pod) still produces series +// to alert on — the etcd_mirror_agent_* family only exists once the agent pod +// runs and is scraped. Served by the manager's /metrics endpoint. +var ( + // allMirrorPhases is the full one-hot domain of etcdMirrorPhaseGauge. + // Pre-emitting every phase at 0/1 means a phase flip never leaves a stale + // 1 series and `== 1` alert expressions are absence-safe once the CR has + // reconciled once. A CR with an empty status.phase emits all zeros. + allMirrorPhases = []ecv1alpha1.EtcdMirrorPhase{ + ecv1alpha1.EtcdMirrorPhasePending, + ecv1alpha1.EtcdMirrorPhaseConnecting, + ecv1alpha1.EtcdMirrorPhaseInitialSync, + ecv1alpha1.EtcdMirrorPhaseSyncing, + ecv1alpha1.EtcdMirrorPhaseDegraded, + ecv1alpha1.EtcdMirrorPhasePaused, + ecv1alpha1.EtcdMirrorPhaseFailed, + } + // conditionStatusValues one-hots each condition across the three + // metav1.ConditionStatus values (lowercased), so `status="true"} == 0` + // distinguishes False/Unknown from series absence. The `type` label + // deliberately mirrors the metav1.Condition field name — NOT + // kube-state-metrics, whose analogous metrics call this label + // `condition`. + conditionStatusValues = []string{"true", "false", "unknown"} + + etcdMirrorPhaseGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "etcd_mirror_phase", + Help: "One-hot mirror of EtcdMirror status.phase (1 on the current phase). " + + "Controller-side: present even when the agent pod never scheduled.", + }, []string{metricLabelNamespace, metricLabelName, "phase"}) + + etcdMirrorConditionGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "etcd_mirror_condition", + Help: "One-hot mirror of EtcdMirror status.conditions per type and status. " + + "Controller-side: updated on every reconcile, including failed agent /statusz polls.", + }, []string{metricLabelNamespace, metricLabelName, "type", "status"}) +) + +func init() { + metrics.Registry.MustRegister(etcdMirrorPhaseGauge, etcdMirrorConditionGauge) +} + +// updateEtcdMirrorMetrics re-emits both gauges from em's current status. +// Conditions iterate what is actually present (types are never removed from +// status once set), so future condition types are covered automatically. +func updateEtcdMirrorMetrics(em *ecv1alpha1.EtcdMirror) { + for _, p := range allMirrorPhases { + var v float64 + if em.Status.Phase == p { + v = 1 + } + etcdMirrorPhaseGauge.WithLabelValues(em.Namespace, em.Name, string(p)).Set(v) + } + for _, c := range em.Status.Conditions { + current := strings.ToLower(string(c.Status)) + for _, s := range conditionStatusValues { + var v float64 + if s == current { + v = 1 + } + etcdMirrorConditionGauge.WithLabelValues(em.Namespace, em.Name, c.Type, s).Set(v) + } + } +} + +// deleteEtcdMirrorMetrics drops every series for one CR — called at CR +// deletion so no stale series outlive it. +func deleteEtcdMirrorMetrics(namespace, name string) { + labels := prometheus.Labels{metricLabelNamespace: namespace, metricLabelName: name} + etcdMirrorPhaseGauge.DeletePartialMatch(labels) + etcdMirrorConditionGauge.DeletePartialMatch(labels) +} diff --git a/internal/controller/etcdmirror_metrics_test.go b/internal/controller/etcdmirror_metrics_test.go new file mode 100644 index 00000000..9c18deeb --- /dev/null +++ b/internal/controller/etcdmirror_metrics_test.go @@ -0,0 +1,129 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +func phaseValue(em *ecv1alpha1.EtcdMirror, phase ecv1alpha1.EtcdMirrorPhase) float64 { + return testutil.ToFloat64(etcdMirrorPhaseGauge.WithLabelValues(em.Namespace, em.Name, string(phase))) +} + +func conditionValue(em *ecv1alpha1.EtcdMirror, condType, status string) float64 { + return testutil.ToFloat64(etcdMirrorConditionGauge.WithLabelValues(em.Namespace, em.Name, condType, status)) +} + +// mirrorSeriesCount counts series of one metric carrying this CR's +// namespace/name labels, straight from the manager registry (the registry is +// process-global and other tests emit series for other CRs). +func mirrorSeriesCount(t *testing.T, metricName, namespace, name string) int { + t.Helper() + mfs, err := ctrlmetrics.Registry.Gather() + require.NoError(t, err) + count := 0 + for _, mf := range mfs { + if mf.GetName() != metricName { + continue + } + for _, m := range mf.GetMetric() { + var nsOK, nameOK bool + for _, lp := range m.GetLabel() { + nsOK = nsOK || (lp.GetName() == "namespace" && lp.GetValue() == namespace) + nameOK = nameOK || (lp.GetName() == "name" && lp.GetValue() == name) + } + if nsOK && nameOK { + count++ + } + } + } + return count +} + +func TestEtcdMirrorMetrics_Lifecycle(t *testing.T) { + em := &ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{Namespace: "metrics-unit-ns", Name: "unit-mirror"}, + } + em.Status.Phase = ecv1alpha1.EtcdMirrorPhaseSyncing + em.Status.Conditions = []metav1.Condition{ + {Type: ecv1alpha1.EtcdMirrorConditionAvailable, Status: metav1.ConditionTrue, Reason: "x"}, + {Type: ecv1alpha1.EtcdMirrorConditionCompacted, Status: metav1.ConditionFalse, Reason: "x"}, + } + updateEtcdMirrorMetrics(em) + + // Phase is one-hot across the full domain. + assert.Equal(t, 1.0, phaseValue(em, ecv1alpha1.EtcdMirrorPhaseSyncing)) + for _, p := range allMirrorPhases { + if p != ecv1alpha1.EtcdMirrorPhaseSyncing { + assert.Zero(t, phaseValue(em, p), "phase %s must be 0", p) + } + } + // Each condition is one-hot across true/false/unknown. + assert.Equal(t, 1.0, conditionValue(em, ecv1alpha1.EtcdMirrorConditionAvailable, "true")) + assert.Zero(t, conditionValue(em, ecv1alpha1.EtcdMirrorConditionAvailable, "false")) + assert.Zero(t, conditionValue(em, ecv1alpha1.EtcdMirrorConditionAvailable, "unknown")) + assert.Equal(t, 1.0, conditionValue(em, ecv1alpha1.EtcdMirrorConditionCompacted, "false")) + assert.Zero(t, conditionValue(em, ecv1alpha1.EtcdMirrorConditionCompacted, "true")) + + // Phase flip leaves no stale 1s. + em.Status.Phase = ecv1alpha1.EtcdMirrorPhaseDegraded + em.Status.Conditions[0].Status = metav1.ConditionUnknown + updateEtcdMirrorMetrics(em) + assert.Zero(t, phaseValue(em, ecv1alpha1.EtcdMirrorPhaseSyncing)) + assert.Equal(t, 1.0, phaseValue(em, ecv1alpha1.EtcdMirrorPhaseDegraded)) + assert.Zero(t, conditionValue(em, ecv1alpha1.EtcdMirrorConditionAvailable, "true")) + assert.Equal(t, 1.0, conditionValue(em, ecv1alpha1.EtcdMirrorConditionAvailable, "unknown")) + + require.Equal(t, len(allMirrorPhases), mirrorSeriesCount(t, "etcd_mirror_phase", em.Namespace, em.Name)) + require.Equal(t, 2*len(conditionStatusValues), mirrorSeriesCount(t, "etcd_mirror_condition", em.Namespace, em.Name)) + + // Deletion drops every series for the CR. + deleteEtcdMirrorMetrics(em.Namespace, em.Name) + assert.Zero(t, mirrorSeriesCount(t, "etcd_mirror_phase", em.Namespace, em.Name)) + assert.Zero(t, mirrorSeriesCount(t, "etcd_mirror_condition", em.Namespace, em.Name)) +} + +func TestEtcdMirrorMetrics_EmptyPhaseAllZeros(t *testing.T) { + em := &ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{Namespace: "metrics-unit-ns", Name: "unreconciled-mirror"}, + } + updateEtcdMirrorMetrics(em) + for _, p := range allMirrorPhases { + assert.Zero(t, phaseValue(em, p)) + } + assert.Zero(t, mirrorSeriesCount(t, "etcd_mirror_condition", em.Namespace, em.Name)) + deleteEtcdMirrorMetrics(em.Namespace, em.Name) +} + +// Registration sanity: both vecs are on the manager registry (linting the +// init() wiring, not prometheus itself). +func TestEtcdMirrorMetrics_Registered(t *testing.T) { + for _, vec := range []*prometheus.GaugeVec{etcdMirrorPhaseGauge, etcdMirrorConditionGauge} { + err := ctrlmetrics.Registry.Register(vec) + var are prometheus.AlreadyRegisteredError + require.ErrorAs(t, err, &are) + } +} diff --git a/internal/controller/etcdmirror_status.go b/internal/controller/etcdmirror_status.go new file mode 100644 index 00000000..c64d9439 --- /dev/null +++ b/internal/controller/etcdmirror_status.go @@ -0,0 +1,428 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "fmt" + "strings" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + "go.etcd.io/etcd-operator/pkg/mirroragent" +) + +const ( + // statusPollInterval drives the /statusz poll for active mirrors (the + // Owns(Deployment) watch supplies the workload edge; status freshness is + // requeue-driven). + statusPollInterval = 15 * time.Second + // slowRequeueInterval is for Paused/Failed/guard-blocked mirrors. The + // agent lingers serving a terminal /statusz, so Failed mirrors keep being + // polled, just slowly. + slowRequeueInterval = time.Minute + + // availableStaleThreshold: the watermark must have advanced within this + // window (3x the agent's progress-notification interval) for Available. + availableStaleThreshold = 3 * mirroragent.DefaultProgressInterval + + // ReplicationLagExceeded's controller-internal constants (v1): the + // watermark more than lagRevisionThreshold revisions behind the source, + // continuously for lagSustainDuration. + lagRevisionThreshold = int64(5000) + lagSustainDuration = 5 * time.Minute +) + +// Condition/validation reasons owned by the controller (the API-contract +// reasons live in api/v1alpha1). +const ( + reasonAgentImageNotConfigured = "AgentImageNotConfigured" + reasonServiceNotFound = "ServiceNotFound" + reasonSecretNotFound = "SecretNotFound" + reasonInvalidTLSSecret = "InvalidTLSSecret" + reasonInvalidAuthSecret = "InvalidAuthSecret" + reasonInvalidConfig = "InvalidConfig" + reasonAgentPodNotReady = "AgentPodNotReady" + reasonAgentStatusUnreachable = "AgentStatusUnreachable" + reasonSnapshotDecodeFailed = "SnapshotDecodeFailed" + reasonPaused = "Paused" + reasonPermanentError = "PermanentError" + reasonDrainComplete = "DrainComplete" + reasonProgressStalled = "ProgressStalled" + reasonWatermarkAdvancing = "WatermarkAdvancing" + reasonConnecting = "Connecting" + reasonInitialSync = "InitialSync" + reasonBackoff = "Backoff" + + reasonNoConflict = "NoConflict" + reasonConflictWinner = "ConflictWinner" + reasonNotThrottled = "NotThrottled" + reasonQuotaOK = "QuotaOK" + reasonSteadyState = "SteadyState" + reasonNoResyncLoop = "NoResyncLoop" + reasonNoDrift = "NoDrift" + reasonInProgress = "InProgress" + reasonComplete = "Complete" + reasonNoViolation = "NoViolation" + reasonNotDrained = "NotDrained" + reasonNoLearner = "NoLearner" + reasonWithinThreshold = "WithinThreshold" + reasonWithinSustainWindow = "WithinSustainWindow" + reasonLagSustained = "LagSustained" + reasonNoVerificationPass = "NoVerificationPass" + reasonStaleVerificationPass = "StaleVerificationPass" + reasonLagExceeded = "LagExceeded" + reasonKeyCountMismatch = "KeyCountMismatch" + reasonDriftDetected = "DriftDetected" + reasonInvariantsHold = "InvariantsHold" + reasonLearnerServing = "LearnerServing" + reasonViolation = "Violation" + reasonConflict = "Conflict" +) + +func setMirrorCondition(em *ecv1alpha1.EtcdMirror, condType string, status metav1.ConditionStatus, reason, message string) { + meta.SetStatusCondition(&em.Status.Conditions, metav1.Condition{ + Type: condType, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: em.Generation, + }) +} + +func setBoolCondition(em *ecv1alpha1.EtcdMirror, condType string, val bool, trueReason, trueMsg, falseReason, falseMsg string) { + if val { + setMirrorCondition(em, condType, metav1.ConditionTrue, trueReason, trueMsg) + } else { + setMirrorCondition(em, condType, metav1.ConditionFalse, falseReason, falseMsg) + } +} + +func metaTimeOrNil(t time.Time) *metav1.Time { + if t.IsZero() { + return nil + } + mt := metav1.NewTime(t) + return &mt +} + +func hexClusterID(id uint64) string { + if id == 0 { + return "" + } + return fmt.Sprintf("%x", id) +} + +// invariantsFreshnessWindow is the InvariantsHeld staleness window: 2x the +// configured periodic reconciliation interval, or 2x the agent's default +// period when the interval is unset or the periodic pass is disabled (counts +// then only refresh on mandatory passes, and the condition goes Unknown once +// the last one ages out). +func invariantsFreshnessWindow(em *ecv1alpha1.EtcdMirror) time.Duration { + rec := em.Spec.Reconciliation + if rec != nil && rec.Enabled && rec.Interval != nil && rec.Interval.Duration > 0 { + return 2 * rec.Interval.Duration + } + return 2 * mirroragent.DefaultReconcilePeriod +} + +// phaseFromSnapshot maps agent phases onto API phases. The library-only +// Drained phase maps to Syncing — the API has no Drained phase, a completed +// drain is not page-worthy, and CutoverReady carries the drain outcome. +func phaseFromSnapshot(p mirroragent.Phase) ecv1alpha1.EtcdMirrorPhase { + if p == mirroragent.PhaseDrained { + return ecv1alpha1.EtcdMirrorPhaseSyncing + } + return ecv1alpha1.EtcdMirrorPhase(p) +} + +// applySnapshotToStatus maps one successfully polled snapshot onto em's +// status fields and conditions. lagSince is the in-memory lag ledger value +// for this CR (zero = not currently over the threshold); the updated value is +// returned for the caller to persist in the ledger. Pure except for em +// mutation — the envtest seam's whole point is that this never dials +// anything. +func applySnapshotToStatus( + em *ecv1alpha1.EtcdMirror, snap *mirroragent.Snapshot, now time.Time, lagSince time.Time, +) time.Time { + st := &em.Status + + st.ObservedGeneration = em.Generation + st.Phase = phaseFromSnapshot(snap.Phase) + st.LastAppliedRevision = snap.Watermark + st.SourceRevision = snap.SourceRevision + // Cluster IDs are latched last-known identities: a fresh agent serves + // zero IDs until its connect-time probe, and blanking the persisted IDs + // in that window would disarm the runtime guard comparisons (sameCluster + // falls back to spec identity), transiently unparking a conflict loser. + if id := hexClusterID(snap.SourceClusterID); id != "" { + st.SourceClusterID = id + } + if id := hexClusterID(snap.TargetClusterID); id != "" { + st.TargetClusterID = id + } + st.SourceVersion = snap.SourceVersion + st.TargetVersion = snap.TargetVersion + // A resumed agent (valid checkpoint -> straight to tail) never sets the + // initialSync* snapshot fields; only a live or completed scan may + // overwrite the persisted values, or a pod restart would wipe them. + if !snap.InitialSyncStartTime.IsZero() || snap.Phase == mirroragent.PhaseInitialSync { + st.InitialSyncKeyCount = snap.InitialSyncKeyCount + st.InitialSyncTotalKeyCount = snap.InitialSyncTotalKeyCount + st.InitialSyncStartTime = metaTimeOrNil(snap.InitialSyncStartTime) + } + if !snap.InitialSyncCompletionTime.IsZero() { + st.InitialSyncCompletionTime = metaTimeOrNil(snap.InitialSyncCompletionTime) + } + st.LeaseBackedKeyCount = snap.LeaseBackedKeyCount + st.ForcedResyncCount = int32(snap.ForcedResyncCount) + st.ScanRestartCount = snap.ScanRestartCount + if !snap.LastReconcileTime.IsZero() { + st.LastReconciliationTime = metaTimeOrNil(snap.LastReconcileTime) + } + if snap.LastReconcileDrift != nil { + st.LastReconciliationDrift = &ecv1alpha1.EtcdMirrorDriftInfo{ + MissingKeys: snap.LastReconcileDrift.MissingKeys, + DivergentKeys: snap.LastReconcileDrift.DivergentKeys, + OrphanKeys: snap.LastReconcileDrift.OrphanKeys, + Repaired: snap.LastReconcileDrift.Repaired, + } + } + if !snap.LastReconcileTime.IsZero() { + st.SourceKeyCount = snap.SourceKeyCount + st.TargetKeyCount = snap.TargetKeyCount + } + if !snap.LastProgressTime.IsZero() { + st.LastProgressTime = metaTimeOrNil(snap.LastProgressTime) + } + switch { + case snap.Cutover != nil: + st.Cutover = &ecv1alpha1.EtcdMirrorCutoverStatus{ + DrainTargetRevision: snap.Cutover.DrainTargetRevision, + DrainedRevision: snap.Cutover.DrainedRevision, + VerifiedTime: metaTimeOrNil(snap.Cutover.VerifiedTime), + SourceKeyCount: snap.Cutover.SourceKeyCount, + TargetKeyCount: snap.Cutover.TargetKeyCount, + LeasedKeyCount: snap.Cutover.LeasedKeyCount, + } + case em.Spec.Mode != ecv1alpha1.EtcdMirrorModeDrain: + // Contract: cutover is populated while spec.mode is Drain. A Drain -> + // Sync flip rolls a fresh agent whose snapshot carries no cutover; + // the stale block must not linger. Mid-drain restarts (mode still + // Drain, cutover transiently absent) retain the last-known values. + st.Cutover = nil + } + st.LastStatusSyncTime = metaTimeOrNil(now) + + lagSince = setLagCondition(em, snap, now, lagSince) + setAvailableCondition(em, snap, now) + setSnapshotFlagConditions(em, snap) + setInvariantsHeldCondition(em, snap, now) + return lagSince +} + +// setLagCondition maintains ReplicationLagExceeded from the raw snapshot +// terms plus the caller's lag ledger: over the revision threshold +// continuously for the sustain window. Both terms come from the same +// watch/progress machinery in the agent. +func setLagCondition( + em *ecv1alpha1.EtcdMirror, snap *mirroragent.Snapshot, now time.Time, lagSince time.Time, +) time.Time { + gap := snap.SourceRevision - snap.Watermark + if gap <= lagRevisionThreshold { + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionReplicationLagExceeded, + metav1.ConditionFalse, reasonWithinThreshold, + fmt.Sprintf("watermark is %d revisions behind the source (threshold %d)", gap, lagRevisionThreshold)) + return time.Time{} + } + if lagSince.IsZero() { + lagSince = now + } + if now.Sub(lagSince) >= lagSustainDuration { + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionReplicationLagExceeded, + metav1.ConditionTrue, reasonLagSustained, + fmt.Sprintf("watermark has been more than %d revisions behind the source for %s (current gap %d); "+ + "note SourceRevision advances on out-of-prefix writes, so the gap overstates lag for prefix-scoped mirrors", + lagRevisionThreshold, now.Sub(lagSince).Round(time.Second), gap)) + } else { + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionReplicationLagExceeded, + metav1.ConditionFalse, reasonWithinSustainWindow, + fmt.Sprintf("watermark gap %d exceeds threshold %d but not yet for the %s sustain window", + gap, lagRevisionThreshold, lagSustainDuration)) + } + return lagSince +} + +func setAvailableCondition(em *ecv1alpha1.EtcdMirror, snap *mirroragent.Snapshot, now time.Time) { + const cond = ecv1alpha1.EtcdMirrorConditionAvailable + switch snap.Phase { + case mirroragent.PhaseDrained: + setMirrorCondition(em, cond, metav1.ConditionTrue, reasonDrainComplete, + "drain completed and verified; the fence role is Primary") + case mirroragent.PhaseSyncing: + if snap.LastProgressTime.IsZero() || now.Sub(snap.LastProgressTime) > availableStaleThreshold { + setMirrorCondition(em, cond, metav1.ConditionFalse, reasonProgressStalled, + fmt.Sprintf("watermark has not advanced within %s", availableStaleThreshold)) + } else { + setMirrorCondition(em, cond, metav1.ConditionTrue, reasonWatermarkAdvancing, + "agent is syncing and the checkpoint watermark is advancing") + } + case mirroragent.PhaseConnecting: + setMirrorCondition(em, cond, metav1.ConditionFalse, reasonConnecting, + "agent is establishing client connections to both sides") + case mirroragent.PhaseInitialSync: + setMirrorCondition(em, cond, metav1.ConditionFalse, reasonInitialSync, + fmt.Sprintf("genesis scan in progress: %d/%d keys", + snap.InitialSyncKeyCount, snap.InitialSyncTotalKeyCount)) + case mirroragent.PhaseDegraded: + setMirrorCondition(em, cond, metav1.ConditionFalse, reasonBackoff, + "agent is in a retry/backoff loop: "+snap.LastError) + case mirroragent.PhaseFailed: + reason := snap.LastErrorReason + if reason == "" { + reason = reasonPermanentError + } + setMirrorCondition(em, cond, metav1.ConditionFalse, reason, snap.LastError) + default: + setMirrorCondition(em, cond, metav1.ConditionUnknown, "UnknownPhase", + fmt.Sprintf("agent reported unknown phase %q", snap.Phase)) + } +} + +// setSnapshotFlagConditions maps the snapshot's condition-shaped flags. +func setSnapshotFlagConditions(em *ecv1alpha1.EtcdMirror, snap *mirroragent.Snapshot) { + setBoolCondition(em, ecv1alpha1.EtcdMirrorConditionTargetThrottled, snap.Throttled, + "Throttled", "target is rejecting the agent's write rate; the agent is backing off "+ + "(raise the target's rate limits or lower sync.maxOpsPerSecond)", + reasonNotThrottled, "target is accepting writes") + setBoolCondition(em, ecv1alpha1.EtcdMirrorConditionTargetQuotaExhausted, snap.QuotaExhausted, + "QuotaExhausted", "target write failed with NOSPACE; compact/defrag/disarm the target "+ + "(backoff cannot heal a full quota — the agent stopped writing)", + reasonQuotaOK, "target storage quota has headroom") + + if snap.Compacted { + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionCompacted, metav1.ConditionTrue, + ecv1alpha1.EtcdMirrorReasonForcedResync, + fmt.Sprintf("forced resync in flight (trigger: %s)", snap.LastResyncReason)) + } else { + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionCompacted, metav1.ConditionFalse, + reasonSteadyState, "no forced resync in flight") + } + + setBoolCondition(em, ecv1alpha1.EtcdMirrorConditionResyncLoopDetected, snap.ResyncLoopDetected, + "ResyncLoop", "consecutive forced resyncs never reached steady state — the livelock signature of "+ + "source compaction retention < scan+drain time; raise retention, raise maxOpsPerSecond, or shrink the prefix", + reasonNoResyncLoop, "no resync livelock detected") + + // DriftDetected is sticky: only recomputed when the agent reports a full + // diff outcome (count-only verifications never overwrite it). + if snap.LastReconcileDrift != nil { + d := snap.LastReconcileDrift + total := d.MissingKeys + d.DivergentKeys + d.OrphanKeys + setBoolCondition(em, ecv1alpha1.EtcdMirrorConditionDriftDetected, total > 0, + reasonDriftDetected, fmt.Sprintf("last reconciliation pass found %d missing, %d divergent, %d orphan keys (repaired: %t)", + d.MissingKeys, d.DivergentKeys, d.OrphanKeys, d.Repaired), + reasonNoDrift, "last reconciliation pass found no drift") + } + + // InitialSyncComplete is durable: the completion time is process memory, + // zeroed by any fresh scan (a Compacted forced resync) and never set by a + // resumed-checkpoint agent (pod restart), yet the contract keeps the + // condition True through both. Only a checkpoint invalidation — a + // cluster-ID mismatch resync or a corrupt checkpoint — resets it. + complete := !snap.InitialSyncCompletionTime.IsZero() + if !complete && meta.IsStatusConditionTrue(em.Status.Conditions, ecv1alpha1.EtcdMirrorConditionInitialSyncComplete) { + invalidated := snap.LastResyncReason == mirroragent.ResyncReasonClusterIDMismatch || + snap.LastErrorReason == ecv1alpha1.EtcdMirrorReasonCheckpointInvalid + complete = !invalidated + } + setBoolCondition(em, ecv1alpha1.EtcdMirrorConditionInitialSyncComplete, complete, + reasonComplete, "genesis scan completed against the checkpointed cluster identities", + reasonInProgress, "genesis scan has not completed") + + violation := snap.Phase == mirroragent.PhaseFailed && snap.LastErrorReason == "EmptyTargetViolation" + setBoolCondition(em, ecv1alpha1.EtcdMirrorConditionEmptyTargetViolation, violation, + reasonViolation, fmt.Sprintf( + "destination prefix was non-empty at genesis with initialSync.mode RequireEmpty. "+ + "To clear the range, run: %s (the reserved checkpoint key is excluded by exact match and recreated by the agent). %s", + etcdctlDelCommand(effectiveDestPrefix(em)), snap.LastError), + reasonNoViolation, "destination prefix passed the RequireEmpty check") + + setBoolCondition(em, ecv1alpha1.EtcdMirrorConditionCutoverReady, snap.CutoverReady, + "CutoverReady", "watermark reached the drain target revision and verification passed; "+ + "the fence role is Primary — safe to repoint clients", + reasonNotDrained, "cutover gate not reached (requires spec.mode Drain, a reached drain target revision, and a verification pass)") + + if snap.SourceLearner || snap.TargetLearner { + var sides []string + if snap.SourceLearner { + sides = append(sides, sideSourceName) + } + if snap.TargetLearner { + sides = append(sides, sideTargetName) + } + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionLearnerEndpoint, metav1.ConditionTrue, + reasonLearnerServing, + fmt.Sprintf("the maintenance Status() probe reported a learner on: %s (non-blocking, but a learner pick can serve stale reads)", + strings.Join(sides, ", "))) + } else { + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionLearnerEndpoint, metav1.ConditionFalse, + reasonNoLearner, "no probed endpoint reported IsLearner") + } + + // Runtime PrefixConflict backstop: the agent refuses (permanently) to + // touch a fence key owned by a different link even when the controller's + // spec-derived guard missed the overlap. + if snap.Phase == mirroragent.PhaseFailed && snap.LastErrorReason == "PrefixConflict" { + setMirrorCondition(em, ecv1alpha1.EtcdMirrorConditionPrefixConflict, metav1.ConditionTrue, + reasonConflict, snap.LastError) + } +} + +// setInvariantsHeldCondition composes the verification verdict: lag ok + +// per-side key counts equal + no drift, from a fresh pass. +func setInvariantsHeldCondition(em *ecv1alpha1.EtcdMirror, snap *mirroragent.Snapshot, now time.Time) { + const cond = ecv1alpha1.EtcdMirrorConditionInvariantsHeld + if snap.LastReconcileTime.IsZero() { + setMirrorCondition(em, cond, metav1.ConditionUnknown, reasonNoVerificationPass, + "no reconciliation/verification pass has run yet (enable spec.reconciliation for a continuous signal)") + return + } + window := invariantsFreshnessWindow(em) + if now.Sub(snap.LastReconcileTime) > window { + setMirrorCondition(em, cond, metav1.ConditionUnknown, reasonStaleVerificationPass, + fmt.Sprintf("last verification pass is older than %s (enable spec.reconciliation for a continuous signal)", window)) + return + } + switch { + case meta.IsStatusConditionTrue(em.Status.Conditions, ecv1alpha1.EtcdMirrorConditionReplicationLagExceeded): + setMirrorCondition(em, cond, metav1.ConditionFalse, reasonLagExceeded, + "ReplicationLagExceeded is True") + case snap.SourceKeyCount != snap.TargetKeyCount: + setMirrorCondition(em, cond, metav1.ConditionFalse, reasonKeyCountMismatch, + fmt.Sprintf("per-side key counts differ: source %d, target %d", snap.SourceKeyCount, snap.TargetKeyCount)) + case meta.IsStatusConditionTrue(em.Status.Conditions, ecv1alpha1.EtcdMirrorConditionDriftDetected): + setMirrorCondition(em, cond, metav1.ConditionFalse, reasonDriftDetected, + "DriftDetected is True") + default: + setMirrorCondition(em, cond, metav1.ConditionTrue, reasonInvariantsHold, + fmt.Sprintf("lag within threshold, per-side key counts equal (%d), no drift; pass fresh within %s", + snap.SourceKeyCount, invariantsFreshnessWindow(em))) + } +} diff --git a/internal/controller/etcdmirror_status_test.go b/internal/controller/etcdmirror_status_test.go new file mode 100644 index 00000000..0c4801e6 --- /dev/null +++ b/internal/controller/etcdmirror_status_test.go @@ -0,0 +1,412 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + "go.etcd.io/etcd-operator/pkg/mirroragent" +) + +func statusFixtureMirror() *ecv1alpha1.EtcdMirror { + em := minimalMirror() + em.Spec.Target.Prefix = "/mirrored/" + return em +} + +func TestInvariantsHeldComposition(t *testing.T) { + now := time.Now() + + apply := func(em *ecv1alpha1.EtcdMirror, snap *mirroragent.Snapshot) { + applySnapshotToStatus(em, snap, now, time.Time{}) + } + + t.Run("fresh equal no-drift is True", func(t *testing.T) { + em := statusFixtureMirror() + apply(em, healthySnapshot()) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionInvariantsHeld, metav1.ConditionTrue, reasonInvariantsHold) + }) + + t.Run("count mismatch is False KeyCountMismatch", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.TargetKeyCount = snap.SourceKeyCount - 1 + apply(em, snap) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionInvariantsHeld, metav1.ConditionFalse, reasonKeyCountMismatch) + }) + + t.Run("sustained lag is False LagExceeded", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.SourceRevision = snap.Watermark + lagRevisionThreshold + 1 + // lagSince predates the sustain window + applySnapshotToStatus(em, snap, now, now.Add(-lagSustainDuration-time.Minute)) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionReplicationLagExceeded, metav1.ConditionTrue, reasonLagSustained) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionInvariantsHeld, metav1.ConditionFalse, reasonLagExceeded) + }) + + t.Run("drift is False DriftDetected", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.LastReconcileDrift = &mirroragent.Drift{OrphanKeys: 2} + apply(em, snap) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionInvariantsHeld, metav1.ConditionFalse, reasonDriftDetected) + }) + + t.Run("stale pass is Unknown StaleVerificationPass", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.LastReconcileTime = now.Add(-2*mirroragent.DefaultReconcilePeriod - time.Minute) + apply(em, snap) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionInvariantsHeld, metav1.ConditionUnknown, reasonStaleVerificationPass) + }) + + t.Run("never ran is Unknown NoVerificationPass", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.LastReconcileTime = time.Time{} + snap.LastReconcileDrift = nil + apply(em, snap) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionInvariantsHeld, metav1.ConditionUnknown, reasonNoVerificationPass) + }) + + t.Run("enabled interval shrinks the freshness window", func(t *testing.T) { + em := statusFixtureMirror() + em.Spec.Reconciliation = &ecv1alpha1.EtcdMirrorReconciliationSpec{ + Enabled: true, + Interval: &metav1.Duration{Duration: 10 * time.Minute}, + } + snap := healthySnapshot() + snap.LastReconcileTime = now.Add(-25 * time.Minute) // > 2x10m, << 2h + apply(em, snap) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionInvariantsHeld, metav1.ConditionUnknown, reasonStaleVerificationPass) + }) + + t.Run("disabled reconciliation uses the 1h default window", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.LastReconcileTime = now.Add(-90 * time.Minute) // < 2h + apply(em, snap) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionInvariantsHeld, metav1.ConditionTrue, reasonInvariantsHold) + }) +} + +func TestConditionsFromSnapshot(t *testing.T) { + now := time.Now() + + t.Run("healthy", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + lag := applySnapshotToStatus(em, snap, now, time.Time{}) + assert.True(t, lag.IsZero()) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionTrue, reasonWatermarkAdvancing) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionTargetThrottled, metav1.ConditionFalse, "") + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionTargetQuotaExhausted, metav1.ConditionFalse, "") + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionCompacted, metav1.ConditionFalse, "") + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionResyncLoopDetected, metav1.ConditionFalse, "") + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionDriftDetected, metav1.ConditionFalse, "") + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionInitialSyncComplete, metav1.ConditionTrue, "") + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionEmptyTargetViolation, metav1.ConditionFalse, "") + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionLearnerEndpoint, metav1.ConditionFalse, "") + assert.Equal(t, ecv1alpha1.EtcdMirrorPhaseSyncing, em.Status.Phase) + assert.Equal(t, fmt.Sprintf("%x", snap.SourceClusterID), em.Status.SourceClusterID) + assert.Equal(t, fmt.Sprintf("%x", snap.TargetClusterID), em.Status.TargetClusterID) + assert.Equal(t, int64(1200), em.Status.LastAppliedRevision) + require.NotNil(t, em.Status.LastStatusSyncTime) + }) + + t.Run("throttled while degraded", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseDegraded + snap.Throttled = true + snap.LastError = "etcdserver: too many requests" + applySnapshotToStatus(em, snap, now, time.Time{}) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionTargetThrottled, metav1.ConditionTrue, "") + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, reasonBackoff) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhaseDegraded, em.Status.Phase) + }) + + t.Run("quota exhausted", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.QuotaExhausted = true + applySnapshotToStatus(em, snap, now, time.Time{}) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionTargetQuotaExhausted, metav1.ConditionTrue, "") + }) + + t.Run("compacted forced resync", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseInitialSync + snap.Compacted = true + snap.LastResyncReason = mirroragent.ResyncReasonCompacted + applySnapshotToStatus(em, snap, now, time.Time{}) + cond := requireCond(t, em, ecv1alpha1.EtcdMirrorConditionCompacted, metav1.ConditionTrue, + ecv1alpha1.EtcdMirrorReasonForcedResync) + assert.Contains(t, cond.Message, "Compacted") + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, reasonInitialSync) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhaseInitialSync, em.Status.Phase) + }) + + t.Run("resync loop", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.ResyncLoopDetected = true + applySnapshotToStatus(em, snap, now, time.Time{}) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionResyncLoopDetected, metav1.ConditionTrue, "") + }) + + t.Run("drift with counts in message", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.LastReconcileDrift = &mirroragent.Drift{MissingKeys: 1, DivergentKeys: 2, OrphanKeys: 3} + applySnapshotToStatus(em, snap, now, time.Time{}) + cond := requireCond(t, em, ecv1alpha1.EtcdMirrorConditionDriftDetected, metav1.ConditionTrue, "") + assert.Contains(t, cond.Message, "1 missing") + assert.Contains(t, cond.Message, "2 divergent") + assert.Contains(t, cond.Message, "3 orphan") + }) + + t.Run("drift condition is sticky when the snapshot carries no diff outcome", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.LastReconcileDrift = &mirroragent.Drift{OrphanKeys: 1} + applySnapshotToStatus(em, snap, now, time.Time{}) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionDriftDetected, metav1.ConditionTrue, "") + + snap2 := healthySnapshot() + snap2.LastReconcileDrift = nil + applySnapshotToStatus(em, snap2, now, time.Time{}) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionDriftDetected, metav1.ConditionTrue, "") + }) + + t.Run("learner endpoint names the side", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.SourceLearner = true + applySnapshotToStatus(em, snap, now, time.Time{}) + cond := requireCond(t, em, ecv1alpha1.EtcdMirrorConditionLearnerEndpoint, metav1.ConditionTrue, "") + assert.Contains(t, cond.Message, "source") + assert.NotContains(t, cond.Message, "target") + }) + + t.Run("failed with typed reason", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseFailed + snap.LastError = "source etcd 3.3.0 below the 3.4 floor" + snap.LastErrorReason = "UnsupportedVersion" + applySnapshotToStatus(em, snap, now, time.Time{}) + cond := requireCond(t, em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, + ecv1alpha1.EtcdMirrorReasonUnsupportedVersion) + assert.Equal(t, snap.LastError, cond.Message) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhaseFailed, em.Status.Phase) + }) + + t.Run("failed untyped falls back to PermanentError", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseFailed + snap.LastError = "some terminal error" + applySnapshotToStatus(em, snap, now, time.Time{}) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, reasonPermanentError) + }) + + t.Run("empty target violation embeds the etcdctl del command", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseFailed + snap.LastError = "destination prefix not empty" + snap.LastErrorReason = "EmptyTargetViolation" + applySnapshotToStatus(em, snap, now, time.Time{}) + cond := requireCond(t, em, ecv1alpha1.EtcdMirrorConditionEmptyTargetViolation, metav1.ConditionTrue, "") + assert.Contains(t, cond.Message, `etcdctl del "/mirrored/" "/mirrored0"`) + assert.Contains(t, cond.Message, "checkpoint key is excluded") + }) + + t.Run("prefix conflict runtime backstop", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseFailed + snap.LastError = "fence key owned by another link" + snap.LastErrorReason = "PrefixConflict" + applySnapshotToStatus(em, snap, now, time.Time{}) + cond := requireCond(t, em, ecv1alpha1.EtcdMirrorConditionPrefixConflict, metav1.ConditionTrue, reasonConflict) + assert.Equal(t, snap.LastError, cond.Message) + }) + + t.Run("drained maps to Syncing plus CutoverReady and DrainComplete", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseDrained + snap.CutoverReady = true + snap.LastProgressTime = now.Add(-time.Hour) // a drained agent stops progressing; still Available + snap.Cutover = &mirroragent.CutoverStatus{ + DrainTargetRevision: 1234, + DrainedRevision: 1234, + VerifiedTime: now, + SourceKeyCount: 500, + TargetKeyCount: 500, + LeasedKeyCount: 3, + } + applySnapshotToStatus(em, snap, now, time.Time{}) + assert.Equal(t, ecv1alpha1.EtcdMirrorPhaseSyncing, em.Status.Phase) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionTrue, reasonDrainComplete) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionCutoverReady, metav1.ConditionTrue, "") + require.NotNil(t, em.Status.Cutover) + assert.Equal(t, int64(1234), em.Status.Cutover.DrainTargetRevision) + assert.Equal(t, int64(3), em.Status.Cutover.LeasedKeyCount) + }) + + t.Run("stalled progress is not Available", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.LastProgressTime = now.Add(-availableStaleThreshold - time.Second) + applySnapshotToStatus(em, snap, now, time.Time{}) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionAvailable, metav1.ConditionFalse, reasonProgressStalled) + }) + + t.Run("lag below sustain window is False WithinSustainWindow and starts the ledger", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.SourceRevision = snap.Watermark + lagRevisionThreshold + 1 + lag := applySnapshotToStatus(em, snap, now, time.Time{}) + assert.Equal(t, now, lag) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionReplicationLagExceeded, + metav1.ConditionFalse, reasonWithinSustainWindow) + }) + + t.Run("key counts only copied when a pass ran", func(t *testing.T) { + em := statusFixtureMirror() + snap := healthySnapshot() + snap.LastReconcileTime = time.Time{} + snap.LastReconcileDrift = nil + applySnapshotToStatus(em, snap, now, time.Time{}) + assert.Zero(t, em.Status.SourceKeyCount) + assert.Zero(t, em.Status.TargetKeyCount) + }) +} + +// restartedAgentSnapshot is what a resumed-from-checkpoint agent process +// serves for its whole life: tailing, with every scan-related field and +// last-resync field at its process-local zero. +func restartedAgentSnapshot(now time.Time) *mirroragent.Snapshot { + snap := healthySnapshot() + snap.InitialSyncKeyCount = 0 + snap.InitialSyncTotalKeyCount = 0 + snap.InitialSyncStartTime = time.Time{} + snap.InitialSyncCompletionTime = time.Time{} + snap.ForcedResyncCount = 0 + snap.ScanRestartCount = 0 + snap.LastResyncReason = "" + snap.LastReconcileTime = time.Time{} + snap.LastReconcileDrift = nil + snap.LastProgressTime = now + return snap +} + +// TestSnapshotRegressionGuards pins the durability contracts: fields the +// agent only knows in process memory must not wipe persisted status when the +// pod restarts, and InitialSyncComplete resets only on checkpoint +// invalidation. +func TestSnapshotRegressionGuards(t *testing.T) { + now := time.Now() + + healthyStatus := func(t *testing.T) *ecv1alpha1.EtcdMirror { + t.Helper() + em := statusFixtureMirror() + applySnapshotToStatus(em, healthySnapshot(), now.Add(-time.Minute), time.Time{}) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionInitialSyncComplete, metav1.ConditionTrue, reasonComplete) + return em + } + + t.Run("pod restart does not regress InitialSyncComplete or wipe initialSync fields", func(t *testing.T) { + em := healthyStatus(t) + wantStart, wantCompletion := em.Status.InitialSyncStartTime, em.Status.InitialSyncCompletionTime + require.NotNil(t, wantCompletion) + + applySnapshotToStatus(em, restartedAgentSnapshot(now), now, time.Time{}) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionInitialSyncComplete, metav1.ConditionTrue, reasonComplete) + assert.Equal(t, wantStart, em.Status.InitialSyncStartTime) + assert.Equal(t, wantCompletion, em.Status.InitialSyncCompletionTime) + assert.Equal(t, int64(500), em.Status.InitialSyncKeyCount) + assert.NotNil(t, em.Status.LastReconciliationTime, "a zero LastReconcileTime must not null the persisted pass time") + }) + + t.Run("Compacted forced resync keeps InitialSyncComplete True", func(t *testing.T) { + em := healthyStatus(t) + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseInitialSync + snap.Compacted = true + snap.LastResyncReason = mirroragent.ResyncReasonCompacted + snap.InitialSyncCompletionTime = time.Time{} // fresh scan zeroes it + snap.InitialSyncKeyCount = 10 + applySnapshotToStatus(em, snap, now, time.Time{}) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionInitialSyncComplete, metav1.ConditionTrue, reasonComplete) + assert.Equal(t, int64(10), em.Status.InitialSyncKeyCount, "a live scan's progress fraction still updates") + }) + + t.Run("cluster-ID mismatch resync resets InitialSyncComplete", func(t *testing.T) { + em := healthyStatus(t) + snap := healthySnapshot() + snap.Phase = mirroragent.PhaseInitialSync + snap.LastResyncReason = mirroragent.ResyncReasonClusterIDMismatch + snap.InitialSyncCompletionTime = time.Time{} + applySnapshotToStatus(em, snap, now, time.Time{}) + requireCond(t, em, ecv1alpha1.EtcdMirrorConditionInitialSyncComplete, metav1.ConditionFalse, reasonInProgress) + }) + + t.Run("Connecting snapshot does not blank the latched cluster IDs", func(t *testing.T) { + em := healthyStatus(t) + srcID, tgtID := em.Status.SourceClusterID, em.Status.TargetClusterID + require.NotEmpty(t, srcID) + + snap := restartedAgentSnapshot(now) + snap.Phase = mirroragent.PhaseConnecting + snap.SourceClusterID, snap.TargetClusterID = 0, 0 + applySnapshotToStatus(em, snap, now, time.Time{}) + assert.Equal(t, srcID, em.Status.SourceClusterID, "zero means 'not yet probed', never 'forget the identity'") + assert.Equal(t, tgtID, em.Status.TargetClusterID) + }) + + t.Run("cutover clears once spec.mode leaves Drain", func(t *testing.T) { + em := healthyStatus(t) + em.Spec.Mode = ecv1alpha1.EtcdMirrorModeDrain + snap := healthySnapshot() + snap.Cutover = &mirroragent.CutoverStatus{DrainTargetRevision: 9, DrainedRevision: 9} + applySnapshotToStatus(em, snap, now, time.Time{}) + require.NotNil(t, em.Status.Cutover) + + // Mid-drain restart (mode still Drain, snapshot briefly without + // cutover): last-known values retained. + applySnapshotToStatus(em, restartedAgentSnapshot(now), now, time.Time{}) + require.NotNil(t, em.Status.Cutover) + + // Drain -> Sync flip: the stale block clears. + em.Spec.Mode = ecv1alpha1.EtcdMirrorModeSync + applySnapshotToStatus(em, restartedAgentSnapshot(now), now, time.Time{}) + assert.Nil(t, em.Status.Cutover) + }) +} diff --git a/internal/controller/etcdmirror_statusz.go b/internal/controller/etcdmirror_statusz.go new file mode 100644 index 00000000..87c61715 --- /dev/null +++ b/internal/controller/etcdmirror_statusz.go @@ -0,0 +1,157 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "net/http" + "time" + + "go.etcd.io/etcd-operator/pkg/mirroragent" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// statuszTimeout bounds one /statusz poll end to end. +const statuszTimeout = 5 * time.Second + +// AgentStatusClient fetches one agent pod's /statusz snapshot. It is an +// injectable seam because envtest has no kubelet: agent pods never run +// there, so tests substitute a fake returning fixture Snapshots. +type AgentStatusClient interface { + Snapshot(ctx context.Context, addr string) (*mirroragent.Snapshot, error) +} + +// httpAgentStatusClient is the production impl: GET http:///statusz +// with a 5s overall timeout, json.Decode into mirroragent.Snapshot (the +// tagged type IS the wire contract — the agent marshals the same Go type). +type httpAgentStatusClient struct{ client *http.Client } + +func newHTTPAgentStatusClient() *httpAgentStatusClient { + return &httpAgentStatusClient{client: &http.Client{Timeout: statuszTimeout}} +} + +func (c *httpAgentStatusClient) Snapshot(ctx context.Context, addr string) (*mirroragent.Snapshot, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+addr+"/statusz", nil) + if err != nil { + return nil, err + } + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("statusz returned %s", resp.Status) + } + var snap mirroragent.Snapshot + if err := json.NewDecoder(resp.Body).Decode(&snap); err != nil { + return nil, &snapshotDecodeError{err: err} + } + return &snap, nil +} + +// snapshotDecodeError distinguishes "reached the agent but its /statusz body +// did not decode" (condition reason SnapshotDecodeFailed) from a transport +// failure (AgentStatusUnreachable). +type snapshotDecodeError struct{ err error } + +func (e *snapshotDecodeError) Error() string { return "decoding /statusz snapshot: " + e.err.Error() } +func (e *snapshotDecodeError) Unwrap() error { return e.err } + +// CheckpointTarget is everything the finalizer needs to reach the target +// etcd and delete the reserved checkpoint key. Credential values flow only +// through this struct into the short-lived client — never into logs, status, +// or events. +type CheckpointTarget struct { + Endpoints []string + // TLS is nil for a cleartext target. + TLS *tls.Config + Username string + Password string + // Key is the reserved checkpoint key (exactly one key, never a prefix). + Key string + // LinkUID is the deleted CR's UID — the fence owner the cleaner requires + // before deleting anything (a PrefixConflict loser's default key is the + // WINNER's live fence; a blind delete would destroy it). + LinkUID string +} + +// CheckpointCleaner deletes the reserved checkpoint key from the target etcd +// during finalization — a short-lived client per call, seamed for envtest +// (no reachable target etcd there). The key is deleted only when its stored +// fence is owned by tgt.LinkUID; an absent key is success, and a foreign or +// undecodable fence is left in place with the reason returned in skipReason +// ("" when the key was deleted or already absent). +type CheckpointCleaner interface { + DeleteCheckpoint(ctx context.Context, tgt CheckpointTarget) (skipReason string, err error) +} + +const ( + checkpointCleanerDialTimeout = 10 * time.Second + checkpointCleanerRequestTimeout = 30 * time.Second +) + +// etcdCheckpointCleaner is the production CheckpointCleaner: one clientv3 +// client per call, an ownership-checked exact-key delete, Close. +type etcdCheckpointCleaner struct{} + +func (etcdCheckpointCleaner) DeleteCheckpoint(ctx context.Context, tgt CheckpointTarget) (string, error) { + cfg := mirroragent.NewClientConfig(tgt.Endpoints, tgt.TLS, checkpointCleanerDialTimeout) + cfg.Username, cfg.Password = tgt.Username, tgt.Password + cfg.Context = ctx + cli, err := clientv3.New(cfg) + if err != nil { + return "", fmt.Errorf("creating target client: %w", err) + } + defer func() { _ = cli.Close() }() + rctx, cancel := context.WithTimeout(ctx, checkpointCleanerRequestTimeout) + defer cancel() + + resp, err := cli.Get(rctx, tgt.Key) + if err != nil { + return "", fmt.Errorf("reading checkpoint key: %w", err) + } + if len(resp.Kvs) == 0 { + return "", nil + } + kv := resp.Kvs[0] + stored, derr := mirroragent.DecodeFenceValue(kv.Value) + if derr != nil { + return "the stored fence is undecodable, so ownership is unprovable; " + + "leaving the key in place (fail-closed, matching the agent)", nil + } + if stored.LinkUID != tgt.LinkUID { + return fmt.Sprintf("the stored fence is owned by link %q, not this mirror; "+ + "leaving the key in place", stored.LinkUID), nil + } + // Mod-revision compare so a fence written between the ownership check and + // the delete (an agent straggler, a takeover) is never deleted blind. + txn, err := cli.Txn(rctx). + If(clientv3.Compare(clientv3.ModRevision(tgt.Key), "=", kv.ModRevision)). + Then(clientv3.OpDelete(tgt.Key)). + Commit() + if err != nil { + return "", fmt.Errorf("deleting checkpoint key: %w", err) + } + if !txn.Succeeded { + return "", fmt.Errorf("checkpoint key moved between ownership check and delete; retrying") + } + return "", nil +} diff --git a/internal/controller/etcdmirror_workload.go b/internal/controller/etcdmirror_workload.go new file mode 100644 index 00000000..421779ea --- /dev/null +++ b/internal/controller/etcdmirror_workload.go @@ -0,0 +1,402 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "maps" + "strconv" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + clientv3 "go.etcd.io/etcd/client/v3" +) + +const ( + // agentHTTPPort is the fixed /statusz-/healthz-/readyz-/metrics port; the + // rendered args pin --http-bind-address to it so the status poller and + // the probes always agree with the binary. + agentHTTPPort = 8080 + + // authSecretsRVAnnotation carries the referenced auth Secrets' + // resourceVersions on the pod template: auth credentials are read once at + // agent startup (unlike TLS material, which reloads per handshake), so an + // auth rotation must roll the pod, and this annotation is what makes the + // rendered template differ. + authSecretsRVAnnotation = "operator.etcd.io/auth-secrets-rv" +) + +const ( + appLabelKey = "app" + controllerLabelKey = "controller" + // defaultClientPortName is EtcdMirrorServiceRef.Port's documented default. + defaultClientPortName = "client" +) + +func deploymentNameForEtcdMirror(em *ecv1alpha1.EtcdMirror) string { + return em.Name + "-mirror-agent" +} + +func etcdMirrorAgentLabels(em *ecv1alpha1.EtcdMirror) map[string]string { + return map[string]string{ + appLabelKey: deploymentNameForEtcdMirror(em), + controllerLabelKey: em.Name, + } +} + +// agentWorkloadInput is everything the Deployment render needs beyond the CR +// itself: resolved endpoints and the presence-only credential layout (never +// secret values). +type agentWorkloadInput struct { + image string + sourceEndpoints string + targetEndpoints string + sourceCreds sideCredsLayout + targetCreds sideCredsLayout +} + +// agentArgs renders the CR spec into the rung-4 mirror-agent flag surface +// (cmd/mirror-agent/config.go is the contract). Field order is fixed and +// excludePrefixes keep spec order, so re-renders of an unchanged spec are +// byte-identical and never cause a spurious rollout. +func agentArgs(em *ecv1alpha1.EtcdMirror, in agentWorkloadInput) []string { + spec := em.Spec + mode := spec.Mode + if mode == "" { + mode = ecv1alpha1.EtcdMirrorModeSync + } + args := []string{ + "--link-uid=" + string(em.UID), + // The epoch is the CR generation: every pod-template-changing + // re-deploy is spec-driven (TLS rotates in place; an auth-annotation + // roll is a legal same-epoch fence resume), and generation is >= 1. + fmt.Sprintf("--epoch=%d", em.Generation), + "--mode=" + string(mode), + } + if spec.Source.Prefix != "" { + args = append(args, "--source-prefix="+spec.Source.Prefix) + } + if spec.Target.Prefix != "" { + args = append(args, "--target-prefix="+spec.Target.Prefix) + } + if spec.Sync.DestPrefix != "" { + args = append(args, "--dest-prefix="+spec.Sync.DestPrefix) + } + for _, p := range spec.Sync.ExcludePrefixes { + args = append(args, "--exclude-prefix="+p) + } + if spec.InitialSync != nil { + if spec.InitialSync.Mode != "" { + args = append(args, "--initial-sync-mode="+string(spec.InitialSync.Mode)) + } + if spec.InitialSync.StartRevision > 0 { + args = append(args, fmt.Sprintf("--start-revision=%d", spec.InitialSync.StartRevision)) + } + } + if spec.Checkpoint != nil && spec.Checkpoint.Key != "" { + args = append(args, "--checkpoint-key="+spec.Checkpoint.Key) + } + if spec.Sync.MaxTxnOps > 0 { + args = append(args, fmt.Sprintf("--max-txn-ops=%d", spec.Sync.MaxTxnOps)) + } + if spec.Sync.TxnFlushBytes != nil { + args = append(args, "--txn-flush-bytes="+spec.Sync.TxnFlushBytes.String()) + } + if spec.Sync.PageKeyLimit > 0 { + args = append(args, fmt.Sprintf("--page-key-limit=%d", spec.Sync.PageKeyLimit)) + } + if spec.Sync.PageBytes != nil { + args = append(args, "--page-bytes="+spec.Sync.PageBytes.String()) + } + if spec.Sync.WatchBufferBytes != nil { + args = append(args, "--watch-buffer-bytes="+spec.Sync.WatchBufferBytes.String()) + } + if spec.Sync.MaxOpsPerSecond > 0 { + args = append(args, fmt.Sprintf("--max-ops-per-second=%d", spec.Sync.MaxOpsPerSecond)) + } + if spec.Sync.RequestTimeout != nil { + args = append(args, "--request-timeout="+spec.Sync.RequestTimeout.Duration.String()) + } + if spec.Sync.DialTimeout != nil { + args = append(args, "--dial-timeout="+spec.Sync.DialTimeout.Duration.String()) + } + if spec.Sync.ReconnectBackoff != nil { + if spec.Sync.ReconnectBackoff.InitialDelay != nil { + args = append(args, "--backoff-initial-delay="+spec.Sync.ReconnectBackoff.InitialDelay.Duration.String()) + } + if spec.Sync.ReconnectBackoff.MaxDelay != nil { + args = append(args, "--backoff-max-delay="+spec.Sync.ReconnectBackoff.MaxDelay.Duration.String()) + } + } + if spec.Reconciliation != nil && spec.Reconciliation.Enabled { + args = append(args, "--reconcile-enabled") + if spec.Reconciliation.Interval != nil { + args = append(args, "--reconcile-interval="+spec.Reconciliation.Interval.Duration.String()) + } + if spec.Reconciliation.DeleteOrphans { + args = append(args, "--reconcile-delete-orphans") + } + } + args = append(args, fmt.Sprintf("--http-bind-address=:%d", agentHTTPPort)) + args = append(args, sideArgs(sideSourceName, spec.Source, in.sourceEndpoints, in.sourceCreds)...) + args = append(args, sideArgs(sideTargetName, spec.Target, in.targetEndpoints, in.targetCreds)...) + return args +} + +const ( + sideSourceName = "source" + sideTargetName = "target" +) + +func sideMountBase(side string) string { return "/etc/mirror-agent/" + side } + +// sideArgs renders one side's flag block. File flags are presence-conditional +// on the resolved Secret layout so the agent never gets a path to a key the +// Secret does not hold. +func sideArgs(side string, ep ecv1alpha1.EtcdMirrorEndpoint, endpoints string, creds sideCredsLayout) []string { + base := sideMountBase(side) + args := []string{"--" + side + "-endpoints=" + endpoints} + if ep.TLS != nil { + args = append(args, "--"+side+"-tls") + if creds.HasClientCert { + args = append(args, + "--"+side+"-cert-file="+base+"/tls/"+tlsSecretCertKey, + "--"+side+"-key-file="+base+"/tls/"+tlsSecretKeyKey) + } + if creds.HasCA { + args = append(args, "--"+side+"-ca-file="+base+"/tls/"+tlsSecretCAKey) + } + if ep.TLS.CABundleRef != nil { + args = append(args, "--"+side+"-ca-bundle-file="+base+"/ca/"+caBundleKey(ep.TLS.CABundleRef)) + } + if ep.TLS.ServerName != "" { + args = append(args, "--"+side+"-server-name="+ep.TLS.ServerName) + } + if ep.TLS.InsecureSkipVerify { + args = append(args, "--"+side+"-insecure-skip-verify") + } + } + if ep.Auth != nil { + args = append(args, + "--"+side+"-username-file="+base+"/auth/"+authUsernameKey, + "--"+side+"-password-file="+base+"/auth/"+authPasswordKey) + } + return args +} + +func caBundleKey(ref *ecv1alpha1.EtcdMirrorCABundleRef) string { + if ref.Key != "" { + return ref.Key + } + return tlsSecretCAKey +} + +// sideVolumes renders one side's Secret/ConfigMap mounts. TLS volumes are +// direct mounts (the agent re-reads them per handshake, so rotation needs no +// pod roll); the auth Secret additionally drives the pod-template +// resourceVersion annotation. +func sideVolumes(side string, ep ecv1alpha1.EtcdMirrorEndpoint) ([]corev1.Volume, []corev1.VolumeMount) { + var vols []corev1.Volume + var mounts []corev1.VolumeMount + base := sideMountBase(side) + // 0440 + the pod-level fsGroup: Secret files stay root-owned, and the + // non-root agent reads them through group ownership. 0400 is unreadable + // for the distroless user (uid 65532) — caught by the TLS e2e. + mode := ptr.To[int32](0o440) + if ep.TLS != nil && ep.TLS.SecretRef != nil { + vols = append(vols, corev1.Volume{ + Name: side + "-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: ep.TLS.SecretRef.Name, + DefaultMode: mode, + }, + }, + }) + mounts = append(mounts, corev1.VolumeMount{Name: side + "-tls", MountPath: base + "/tls", ReadOnly: true}) + } + if ep.TLS != nil && ep.TLS.CABundleRef != nil { + ref := ep.TLS.CABundleRef + vol := corev1.Volume{Name: side + "-ca-bundle"} + if ref.Kind == "Secret" { + vol.VolumeSource = corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: ref.Name, DefaultMode: mode}, + } + } else { + vol.VolumeSource = corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: ref.Name}, + DefaultMode: mode, + }, + } + } + vols = append(vols, vol) + mounts = append(mounts, corev1.VolumeMount{Name: side + "-ca-bundle", MountPath: base + "/ca", ReadOnly: true}) + } + if ep.Auth != nil { + vols = append(vols, corev1.Volume{ + Name: side + "-auth", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: ep.Auth.SecretRef.Name, + DefaultMode: mode, + }, + }, + }) + mounts = append(mounts, corev1.VolumeMount{Name: side + "-auth", MountPath: base + "/auth", ReadOnly: true}) + } + return vols, mounts +} + +// renderAgentDeployment renders the size-1 stateless agent Deployment. +// Strategy is Recreate: the fence tolerates an overlap window, but a dual +// writer on every spec rollout buys nothing. +func renderAgentDeployment(em *ecv1alpha1.EtcdMirror, in agentWorkloadInput, replicas int32) *appsv1.Deployment { + labels := etcdMirrorAgentLabels(em) + + srcVols, srcMounts := sideVolumes(sideSourceName, em.Spec.Source) + tgtVols, tgtMounts := sideVolumes(sideTargetName, em.Spec.Target) + + container := corev1.Container{ + Name: "mirror-agent", + // The operator image ships the binary at /mirror-agent but its + // ENTRYPOINT is /manager, so the command must be explicit. + Command: []string{"/mirror-agent"}, + Args: agentArgs(em, in), + Image: in.image, + Ports: []corev1.ContainerPort{{Name: "http", ContainerPort: agentHTTPPort}}, + VolumeMounts: append(srcMounts, tgtMounts...), + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromString("http")}, + }, + PeriodSeconds: 10, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstr.FromString("http")}, + }, + }, + } + if em.Spec.Resources != nil { + container.Resources = *em.Spec.Resources + } + + podSpec := corev1.PodSpec{ + Containers: []corev1.Container{container}, + Volumes: append(srcVols, tgtVols...), + // The agent has zero Kubernetes API dependency; don't hand it a token. + AutomountServiceAccountToken: ptr.To(false), + // The agent image runs as a non-root distroless user; Secret volume + // files are root-owned, so group ownership (fsGroup) plus the 0440 + // mount mode is what makes the mounted TLS/auth material readable. + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To[int64](65532), + }, + } + + podMeta := metav1.ObjectMeta{ + Labels: make(map[string]string), + Annotations: make(map[string]string), + } + if em.Spec.PodTemplate != nil && em.Spec.PodTemplate.Spec != nil { + podSpec.Affinity = em.Spec.PodTemplate.Spec.Affinity + podSpec.NodeSelector = em.Spec.PodTemplate.Spec.NodeSelector + podSpec.Tolerations = em.Spec.PodTemplate.Spec.Tolerations + } + if em.Spec.PodTemplate != nil && em.Spec.PodTemplate.Metadata != nil { + maps.Copy(podMeta.Labels, em.Spec.PodTemplate.Metadata.Labels) + maps.Copy(podMeta.Annotations, em.Spec.PodTemplate.Metadata.Annotations) + } + maps.Copy(podMeta.Labels, labels) + if em.Spec.Source.Auth != nil || em.Spec.Target.Auth != nil { + podMeta.Annotations[authSecretsRVAnnotation] = + in.sourceCreds.AuthSecretRV + "/" + in.targetCreds.AuthSecretRV + } + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentNameForEtcdMirror(em), + Namespace: em.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(replicas), + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Strategy: appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: podMeta, + Spec: podSpec, + }, + }, + } +} + +// serviceRefEndpoints resolves a serviceRef to one schemeless +// "..svc.cluster.local:" endpoint. The port may be a name +// ("client" by default), which cannot appear in a dial URL, so a named port +// is resolved against the Service's spec. +func serviceRefEndpoints( + ctx context.Context, c client.Client, ns string, ref *ecv1alpha1.EtcdMirrorServiceRef, +) (string, error) { + svcNS := ref.Namespace + if svcNS == "" { + svcNS = ns + } + portSpec := ref.Port + if portSpec == "" { + portSpec = defaultClientPortName + } + svc := &corev1.Service{} + if err := c.Get(ctx, types.NamespacedName{Namespace: svcNS, Name: ref.Name}, svc); err != nil { + if apierrors.IsNotFound(err) { + return "", &credsError{Reason: reasonServiceNotFound, + msg: fmt.Sprintf("serviceRef Service %s/%s not found", svcNS, ref.Name)} + } + return "", err + } + if n, err := strconv.Atoi(portSpec); err == nil { + return fmt.Sprintf("%s.%s.svc.cluster.local:%d", ref.Name, svcNS, n), nil + } + for _, p := range svc.Spec.Ports { + if p.Name == portSpec { + return fmt.Sprintf("%s.%s.svc.cluster.local:%d", ref.Name, svcNS, p.Port), nil + } + } + return "", &credsError{Reason: reasonServiceNotFound, + msg: fmt.Sprintf("Service %s/%s has no port named %q", svcNS, ref.Name, portSpec)} +} + +// etcdctlDelCommand renders the exact recovery command for an +// EmptyTargetViolation: the offending effective destination range as etcd +// sees it. The empty prefix is the whole keyspace, which has no prefix range +// end — only the --from-key form expresses it. +func etcdctlDelCommand(effectiveDestPrefix string) string { + if effectiveDestPrefix == "" { + return `etcdctl del "" --from-key` + } + return fmt.Sprintf("etcdctl del %q %q", effectiveDestPrefix, clientv3.GetPrefixRangeEnd(effectiveDestPrefix)) +} diff --git a/internal/controller/etcdmirror_workload_test.go b/internal/controller/etcdmirror_workload_test.go new file mode 100644 index 00000000..54a05dae --- /dev/null +++ b/internal/controller/etcdmirror_workload_test.go @@ -0,0 +1,289 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +func minimalMirror() *ecv1alpha1.EtcdMirror { + return &ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{ + Name: "m1", + Namespace: "default", + UID: types.UID("uid-1"), + Generation: 3, + }, + Spec: ecv1alpha1.EtcdMirrorSpec{ + Source: ecv1alpha1.EtcdMirrorEndpoint{EndpointList: []string{"src:2379"}}, + Target: ecv1alpha1.EtcdMirrorEndpoint{EndpointList: []string{"tgt:2379"}}, + }, + } +} + +func minimalInput() agentWorkloadInput { + return agentWorkloadInput{ + image: testAgentImage, + sourceEndpoints: "src:2379", + targetEndpoints: "tgt:2379", + } +} + +func TestMirrorAgentArgs(t *testing.T) { + t.Run("minimal spec emits only always-flags", func(t *testing.T) { + got := agentArgs(minimalMirror(), minimalInput()) + want := []string{ + "--link-uid=uid-1", + "--epoch=3", + "--mode=Sync", + "--http-bind-address=:8080", + "--source-endpoints=src:2379", + "--target-endpoints=tgt:2379", + } + assert.Equal(t, want, got) + }) + + t.Run("drain mode is explicit", func(t *testing.T) { + em := minimalMirror() + em.Spec.Mode = ecv1alpha1.EtcdMirrorModeDrain + assert.Contains(t, agentArgs(em, minimalInput()), "--mode=Drain") + }) + + t.Run("full spec emits the exact ordered slice", func(t *testing.T) { + em := minimalMirror() + em.Spec.Source.Prefix = "/registry/" + em.Spec.Target.Prefix = "/mirrored/" + em.Spec.Source.TLS = &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: &corev1.LocalObjectReference{Name: "src-tls"}, + CABundleRef: &ecv1alpha1.EtcdMirrorCABundleRef{Name: "trust", Key: "bundle.pem"}, + ServerName: "etcd.example.com", + } + em.Spec.Source.Auth = &ecv1alpha1.EtcdMirrorAuth{ + SecretRef: corev1.LocalObjectReference{Name: "src-auth"}, + } + em.Spec.Target.TLS = &ecv1alpha1.EtcdMirrorTLS{ + InsecureSkipVerify: true, + InsecureSkipVerifyAcknowledgeRisk: true, + } + em.Spec.Sync = ecv1alpha1.EtcdMirrorSyncSpec{ + DestPrefix: "sub/", + ExcludePrefixes: []string{"/registry/events/", "/registry/leases/"}, + MaxTxnOps: 64, + TxnFlushBytes: resource.NewQuantity(512*1024, resource.BinarySI), + PageKeyLimit: 256, + PageBytes: ptrQuantity("2Mi"), + WatchBufferBytes: ptrQuantity("32Mi"), + MaxOpsPerSecond: 1000, + RequestTimeout: &metav1.Duration{Duration: 20 * time.Second}, + DialTimeout: &metav1.Duration{Duration: 5 * time.Second}, + ReconnectBackoff: &ecv1alpha1.EtcdMirrorBackoffSpec{ + InitialDelay: &metav1.Duration{Duration: 2 * time.Second}, + MaxDelay: &metav1.Duration{Duration: time.Minute}, + }, + } + em.Spec.InitialSync = &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwrite, + StartRevision: 42, + } + em.Spec.Checkpoint = &ecv1alpha1.EtcdMirrorCheckpointSpec{Key: "/mirrored/sub/\x00cp"} + em.Spec.Reconciliation = &ecv1alpha1.EtcdMirrorReconciliationSpec{ + Enabled: true, + Interval: &metav1.Duration{Duration: 30 * time.Minute}, + DeleteOrphans: true, + } + in := minimalInput() + in.sourceCreds = sideCredsLayout{HasClientCert: true, HasCA: true, HasAuth: true, AuthSecretRV: "7"} + + want := []string{ + "--link-uid=uid-1", + "--epoch=3", + "--mode=Sync", + "--source-prefix=/registry/", + "--target-prefix=/mirrored/", + "--dest-prefix=sub/", + "--exclude-prefix=/registry/events/", + "--exclude-prefix=/registry/leases/", + "--initial-sync-mode=Overwrite", + "--start-revision=42", + "--checkpoint-key=/mirrored/sub/\x00cp", + "--max-txn-ops=64", + "--txn-flush-bytes=512Ki", + "--page-key-limit=256", + "--page-bytes=2Mi", + "--watch-buffer-bytes=32Mi", + "--max-ops-per-second=1000", + "--request-timeout=20s", + "--dial-timeout=5s", + "--backoff-initial-delay=2s", + "--backoff-max-delay=1m0s", + "--reconcile-enabled", + "--reconcile-interval=30m0s", + "--reconcile-delete-orphans", + "--http-bind-address=:8080", + "--source-endpoints=src:2379", + "--source-tls", + "--source-cert-file=/etc/mirror-agent/source/tls/tls.crt", + "--source-key-file=/etc/mirror-agent/source/tls/tls.key", + "--source-ca-file=/etc/mirror-agent/source/tls/ca.crt", + "--source-ca-bundle-file=/etc/mirror-agent/source/ca/bundle.pem", + "--source-server-name=etcd.example.com", + "--source-username-file=/etc/mirror-agent/source/auth/username", + "--source-password-file=/etc/mirror-agent/source/auth/password", + "--target-endpoints=tgt:2379", + "--target-tls", + "--target-insecure-skip-verify", + } + assert.Equal(t, want, agentArgs(em, in)) + }) + + t.Run("checkpoint key flag only when set", func(t *testing.T) { + em := minimalMirror() + em.Spec.Checkpoint = &ecv1alpha1.EtcdMirrorCheckpointSpec{} + for _, a := range agentArgs(em, minimalInput()) { + assert.NotContains(t, a, "--checkpoint-key") + } + }) + + t.Run("start revision only when positive", func(t *testing.T) { + em := minimalMirror() + em.Spec.InitialSync = &ecv1alpha1.EtcdMirrorInitialSyncSpec{Mode: ecv1alpha1.EtcdMirrorInitialSyncRequireEmpty} + for _, a := range agentArgs(em, minimalInput()) { + assert.NotContains(t, a, "--start-revision") + } + }) + + t.Run("deterministic across renders", func(t *testing.T) { + em := minimalMirror() + em.Spec.Sync.ExcludePrefixes = []string{"/b/", "/a/"} + first := agentArgs(em, minimalInput()) + second := agentArgs(em, minimalInput()) + assert.Equal(t, first, second) + // excludePrefixes keep spec order, not sorted order + assert.Contains(t, first, "--exclude-prefix=/b/") + i1, i2 := indexOf(first, "--exclude-prefix=/b/"), indexOf(first, "--exclude-prefix=/a/") + assert.Less(t, i1, i2) + }) +} + +func ptrQuantity(s string) *resource.Quantity { + q := resource.MustParse(s) + return &q +} + +func indexOf(ss []string, want string) int { + for i, s := range ss { + if s == want { + return i + } + } + return -1 +} + +func TestMirrorDeploymentRender(t *testing.T) { + em := minimalMirror() + em.Spec.Source.TLS = &ecv1alpha1.EtcdMirrorTLS{SecretRef: &corev1.LocalObjectReference{Name: "src-tls"}} + em.Spec.Source.Auth = &ecv1alpha1.EtcdMirrorAuth{SecretRef: corev1.LocalObjectReference{Name: "src-auth"}} + em.Spec.Resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("128Mi")}, + } + em.Spec.PodTemplate = &ecv1alpha1.PodTemplate{ + Metadata: &ecv1alpha1.PodMetadata{ + Labels: map[string]string{"team": "storage"}, + Annotations: map[string]string{"custom": "anno"}, + }, + Spec: &ecv1alpha1.PodSpec{ + NodeSelector: map[string]string{"zone": "z1"}, + Tolerations: []corev1.Toleration{{Key: "dedicated", Operator: corev1.TolerationOpExists}}, + }, + } + in := minimalInput() + in.sourceCreds = sideCredsLayout{HasClientCert: false, HasCA: true, HasAuth: true, AuthSecretRV: "41"} + + dep := renderAgentDeployment(em, in, 1) + + assert.Equal(t, "m1-mirror-agent", dep.Name) + assert.Equal(t, int32(1), *dep.Spec.Replicas) + assert.Equal(t, "Recreate", string(dep.Spec.Strategy.Type)) + assert.Equal(t, map[string]string{"app": "m1-mirror-agent", "controller": "m1"}, dep.Spec.Selector.MatchLabels) + + pod := dep.Spec.Template + require.Len(t, pod.Spec.Containers, 1) + c := pod.Spec.Containers[0] + assert.Equal(t, []string{"/mirror-agent"}, c.Command) + assert.Equal(t, testAgentImage, c.Image) + require.Len(t, c.Ports, 1) + assert.Equal(t, int32(8080), c.Ports[0].ContainerPort) + assert.Equal(t, "/readyz", c.ReadinessProbe.HTTPGet.Path) + assert.Equal(t, "/healthz", c.LivenessProbe.HTTPGet.Path) + assert.Equal(t, resource.MustParse("128Mi"), c.Resources.Requests[corev1.ResourceMemory]) + + // The secret lacks tls.crt: no cert-file flags, but the ca-file flag stays. + assert.NotContains(t, c.Args, "--source-cert-file=/etc/mirror-agent/source/tls/tls.crt") + assert.Contains(t, c.Args, "--source-ca-file=/etc/mirror-agent/source/tls/ca.crt") + + require.NotNil(t, pod.Spec.AutomountServiceAccountToken) + assert.False(t, *pod.Spec.AutomountServiceAccountToken) + + // podTemplate propagation plus reserved labels winning + assert.Equal(t, "storage", pod.Labels["team"]) + assert.Equal(t, "m1-mirror-agent", pod.Labels["app"]) + assert.Equal(t, "anno", pod.Annotations["custom"]) + assert.Equal(t, map[string]string{"zone": "z1"}, pod.Spec.NodeSelector) + require.Len(t, pod.Spec.Tolerations, 1) + + // auth rotation annotation (source RV, empty target RV) + assert.Equal(t, "41/", pod.Annotations[authSecretsRVAnnotation]) + + // volumes: source tls + source auth, 0440 + names := map[string]corev1.Volume{} + for _, v := range pod.Spec.Volumes { + names[v.Name] = v + } + require.Contains(t, names, "source-tls") + require.Contains(t, names, "source-auth") + assert.NotContains(t, names, "target-tls") + assert.Equal(t, int32(0o440), *names["source-tls"].Secret.DefaultMode) + assert.Equal(t, "src-tls", names["source-tls"].Secret.SecretName) + + // non-root readability: fsGroup pairs with the 0440 mount mode + require.NotNil(t, pod.Spec.SecurityContext) + assert.Equal(t, int64(65532), *pod.Spec.SecurityContext.FSGroup) + + // paused render + pausedDep := renderAgentDeployment(em, in, 0) + assert.Equal(t, int32(0), *pausedDep.Spec.Replicas) + + // no auth annotation when neither side uses auth + em2 := minimalMirror() + dep2 := renderAgentDeployment(em2, minimalInput(), 1) + assert.NotContains(t, dep2.Spec.Template.Annotations, authSecretsRVAnnotation) +} + +func TestEtcdctlDelCommandMessage(t *testing.T) { + assert.Equal(t, `etcdctl del "/mirrored/" "/mirrored0"`, etcdctlDelCommand("/mirrored/")) + assert.Equal(t, `etcdctl del "" --from-key`, etcdctlDelCommand("")) +} diff --git a/internal/controller/mirror_creds.go b/internal/controller/mirror_creds.go new file mode 100644 index 00000000..5ad3583d --- /dev/null +++ b/internal/controller/mirror_creds.go @@ -0,0 +1,354 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "strings" + "time" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + "go.etcd.io/etcd-operator/pkg/mirroragent" +) + +// SECURITY: this file is the single seam through which EtcdMirror credential +// Secrets are read. The discipline (the objectstore credential-handling +// precedent): +// +// - Log only namespace/side/secret NAMES — never a key's value, never +// certificate or credential bytes. +// - Errors wrap only object names, key names, and API errors — never data. +// - No secret VALUE is ever returned to the render/status/event paths: +// rendering needs only presence booleans (sideCredsLayout); the raw +// bytes flow exclusively into resolveFinalizerTarget's in-memory +// CheckpointTarget for the short-lived finalizer client. + +const ( + tlsSecretCertKey = "tls.crt" + tlsSecretKeyKey = "tls.key" + tlsSecretCAKey = "ca.crt" + authUsernameKey = "username" + authPasswordKey = "password" +) + +// credsError is a validation failure with an API-facing condition reason. +// Its message carries object/key names only, never secret data. +type credsError struct { + Reason string + msg string +} + +func (e *credsError) Error() string { return e.msg } + +// sideCredsLayout is the presence-only view of one side's credential +// material, everything the Deployment render needs (file flags are +// presence-conditional) without ever seeing a byte of it. +type sideCredsLayout struct { + HasClientCert bool + HasCA bool + HasAuth bool + // AuthSecretRV is the auth Secret's resourceVersion, rolled into a pod + // template annotation so an auth rotation restarts the pod (auth is read + // once at agent startup; TLS reloads live and needs no roll). + AuthSecretRV string +} + +// resolveSideCreds validates one side's referenced Secrets/ConfigMaps and +// returns their presence layout. Never logs or returns secret values. +func resolveSideCreds( + ctx context.Context, c client.Client, logger logr.Logger, + ns, side string, ep ecv1alpha1.EtcdMirrorEndpoint, +) (sideCredsLayout, error) { + var layout sideCredsLayout + if ep.TLS != nil && ep.TLS.SecretRef != nil { + name := ep.TLS.SecretRef.Name + secret := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, secret); err != nil { + if apierrors.IsNotFound(err) { + return layout, &credsError{Reason: reasonSecretNotFound, + msg: fmt.Sprintf("%s TLS secret %s/%s not found", side, ns, name)} + } + return layout, err + } + _, hasCert := secret.Data[tlsSecretCertKey] + _, hasKey := secret.Data[tlsSecretKeyKey] + if hasCert != hasKey { + return layout, &credsError{Reason: reasonInvalidTLSSecret, + msg: fmt.Sprintf("%s TLS secret %s/%s has one of %s/%s without the other", + side, ns, name, tlsSecretCertKey, tlsSecretKeyKey)} + } + layout.HasClientCert = hasCert + _, layout.HasCA = secret.Data[tlsSecretCAKey] + logger.V(1).Info("resolved TLS secret", "side", side, "secret", name, + "hasClientCert", layout.HasClientCert, "hasCA", layout.HasCA) + } + if ep.TLS != nil && ep.TLS.CABundleRef != nil { + if _, err := readCABundle(ctx, c, ns, side, ep.TLS.CABundleRef); err != nil { + return layout, err + } + } + if ep.Auth != nil { + name := ep.Auth.SecretRef.Name + secret := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, secret); err != nil { + if apierrors.IsNotFound(err) { + return layout, &credsError{Reason: reasonSecretNotFound, + msg: fmt.Sprintf("%s auth secret %s/%s not found", side, ns, name)} + } + return layout, err + } + if len(secret.Data[authUsernameKey]) == 0 || len(secret.Data[authPasswordKey]) == 0 { + return layout, &credsError{Reason: reasonInvalidAuthSecret, + msg: fmt.Sprintf("%s auth secret %s/%s must hold non-empty %q and %q keys", + side, ns, name, authUsernameKey, authPasswordKey)} + } + layout.HasAuth = true + layout.AuthSecretRV = secret.ResourceVersion + logger.V(1).Info("resolved auth secret", "side", side, "secret", name) + } + return layout, nil +} + +// readCABundle fetches the PEM trust bundle a caBundleRef points at (Secret +// or ConfigMap). The returned bytes are used only by resolveFinalizerTarget; +// validation callers discard them. +func readCABundle( + ctx context.Context, c client.Client, ns, side string, ref *ecv1alpha1.EtcdMirrorCABundleRef, +) ([]byte, error) { + key := ref.Key + if key == "" { + key = tlsSecretCAKey + } + kind := ref.Kind + if kind == "" { + kind = "ConfigMap" + } + nn := types.NamespacedName{Namespace: ns, Name: ref.Name} + var data []byte + var found bool + if kind == "Secret" { + secret := &corev1.Secret{} + if err := c.Get(ctx, nn, secret); err != nil { + if apierrors.IsNotFound(err) { + return nil, &credsError{Reason: reasonSecretNotFound, + msg: fmt.Sprintf("%s caBundleRef Secret %s/%s not found", side, ns, ref.Name)} + } + return nil, err + } + data, found = secret.Data[key] + } else { + cm := &corev1.ConfigMap{} + if err := c.Get(ctx, nn, cm); err != nil { + if apierrors.IsNotFound(err) { + return nil, &credsError{Reason: reasonSecretNotFound, + msg: fmt.Sprintf("%s caBundleRef ConfigMap %s/%s not found", side, ns, ref.Name)} + } + return nil, err + } + var s string + s, found = cm.Data[key] + data = []byte(s) + } + if !found { + return nil, &credsError{Reason: reasonInvalidTLSSecret, + msg: fmt.Sprintf("%s caBundleRef %s %s/%s has no key %q", side, kind, ns, ref.Name, key)} + } + return data, nil +} + +// checkpointKeyForMirror mirrors the agent's checkpoint-key defaulting +// (cmd/mirror-agent passes --checkpoint-key only when spec.checkpoint.key is +// set; the engine default is prefix + DefaultCheckpointKeySuffix). +func checkpointKeyForMirror(em *ecv1alpha1.EtcdMirror) string { + if em.Spec.Checkpoint != nil && em.Spec.Checkpoint.Key != "" { + return em.Spec.Checkpoint.Key + } + return effectiveDestPrefix(em) + mirroragent.DefaultCheckpointKeySuffix +} + +// resolveFinalizerTarget builds the in-memory dial material for the +// finalizer's one-key delete. Secret values flow only into the returned +// struct — same never-log contract as resolveSideCreds. +func resolveFinalizerTarget( + ctx context.Context, c client.Client, logger logr.Logger, em *ecv1alpha1.EtcdMirror, +) (CheckpointTarget, error) { + tgt := CheckpointTarget{Key: checkpointKeyForMirror(em), LinkUID: string(em.UID)} + ep := em.Spec.Target + + switch { + case len(ep.EndpointList) > 0: + tgt.Endpoints = ep.EndpointList + case ep.ServiceRef != nil: + addr, err := serviceRefEndpoints(ctx, c, em.Namespace, ep.ServiceRef) + if err != nil { + return CheckpointTarget{}, err + } + scheme := "http://" + if ep.TLS != nil { + scheme = "https://" + } + tgt.Endpoints = []string{scheme + addr} + default: + return CheckpointTarget{}, &credsError{Reason: reasonInvalidConfig, + msg: "target has neither endpointList nor serviceRef"} + } + + if ep.TLS != nil { + tlsCfg, err := finalizerTLSConfig(ctx, c, em.Namespace, ep.TLS) + if err != nil { + return CheckpointTarget{}, err + } + tgt.TLS = tlsCfg + } + + if ep.Auth != nil { + name := ep.Auth.SecretRef.Name + secret := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: em.Namespace, Name: name}, secret); err != nil { + return CheckpointTarget{}, fmt.Errorf("reading target auth secret %s/%s: %w", em.Namespace, name, err) + } + tgt.Username = strings.TrimRight(string(secret.Data[authUsernameKey]), "\r\n") + tgt.Password = strings.TrimRight(string(secret.Data[authPasswordKey]), "\r\n") + } + logger.V(1).Info("resolved finalizer target", "endpoints", tgt.Endpoints, "key", tgt.Key) + return tgt, nil +} + +// certExpiry names one piece of referenced TLS material and when it expires. +// Only the side, a human-readable kind, and NotAfter escape this file — never +// certificate bytes. +type certExpiry struct { + Side string + Kind string + NotAfter time.Time +} + +// earliestNotAfter parses every CERTIFICATE block in pemData and returns the +// earliest NotAfter (zero when nothing parses — best-effort: expiry warnings +// must never fail a reconcile, and parse errors are surfaced by the agent's +// own TLS handshake failures). +func earliestNotAfter(pemData []byte) time.Time { + var earliest time.Time + for len(pemData) > 0 { + var block *pem.Block + block, pemData = pem.Decode(pemData) + if block == nil { + break + } + if block.Type != "CERTIFICATE" { + continue + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + continue + } + if earliest.IsZero() || cert.NotAfter.Before(earliest) { + earliest = cert.NotAfter + } + } + return earliest +} + +// sideCertExpiries collects the NotAfter of each cert material one side +// references (client leaf, ca.crt, caBundle). Best-effort: unreadable objects +// or unparseable PEM yield no entry rather than an error. +func sideCertExpiries( + ctx context.Context, c client.Client, ns, side string, ep ecv1alpha1.EtcdMirrorEndpoint, +) []certExpiry { + if ep.TLS == nil { + return nil + } + var out []certExpiry + if ep.TLS.SecretRef != nil { + secret := &corev1.Secret{} + nn := types.NamespacedName{Namespace: ns, Name: ep.TLS.SecretRef.Name} + if err := c.Get(ctx, nn, secret); err == nil { + if t := earliestNotAfter(secret.Data[tlsSecretCertKey]); !t.IsZero() { + out = append(out, certExpiry{Side: side, Kind: "client certificate (tls.crt)", NotAfter: t}) + } + if t := earliestNotAfter(secret.Data[tlsSecretCAKey]); !t.IsZero() { + out = append(out, certExpiry{Side: side, Kind: "CA (ca.crt)", NotAfter: t}) + } + } + } + if ep.TLS.CABundleRef != nil { + if data, err := readCABundle(ctx, c, ns, side, ep.TLS.CABundleRef); err == nil { + if t := earliestNotAfter(data); !t.IsZero() { + out = append(out, certExpiry{Side: side, Kind: "CA bundle (caBundleRef)", NotAfter: t}) + } + } + } + return out +} + +// finalizerTLSConfig assembles a one-shot *tls.Config from the referenced +// Secret bytes (the finalizer client lives seconds; no live reload needed). +func finalizerTLSConfig( + ctx context.Context, c client.Client, ns string, spec *ecv1alpha1.EtcdMirrorTLS, +) (*tls.Config, error) { + cfg := &tls.Config{ + MinVersion: tls.VersionTLS12, + ServerName: spec.ServerName, + InsecureSkipVerify: spec.InsecureSkipVerify, // #nosec G402 -- CR opted in via acknowledged spec field + } + if spec.SecretRef != nil { + secret := &corev1.Secret{} + nn := types.NamespacedName{Namespace: ns, Name: spec.SecretRef.Name} + if err := c.Get(ctx, nn, secret); err != nil { + return nil, fmt.Errorf("reading target TLS secret %s/%s: %w", ns, spec.SecretRef.Name, err) + } + certPEM, keyPEM := secret.Data[tlsSecretCertKey], secret.Data[tlsSecretKeyKey] + if len(certPEM) > 0 && len(keyPEM) > 0 { + pair, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, fmt.Errorf("parsing client certificate in secret %s/%s: %w", + ns, spec.SecretRef.Name, err) + } + cfg.Certificates = []tls.Certificate{pair} + } + if caPEM := secret.Data[tlsSecretCAKey]; len(caPEM) > 0 && spec.CABundleRef == nil { + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("no CA certificate parsed from secret %s/%s key %s", + ns, spec.SecretRef.Name, tlsSecretCAKey) + } + cfg.RootCAs = pool + } + } + if spec.CABundleRef != nil { + caPEM, err := readCABundle(ctx, c, ns, "target", spec.CABundleRef) + if err != nil { + return nil, err + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("no CA certificate parsed from caBundleRef %s", spec.CABundleRef.Name) + } + cfg.RootCAs = pool + } + return cfg, nil +} diff --git a/internal/controller/mirror_creds_test.go b/internal/controller/mirror_creds_test.go new file mode 100644 index 00000000..141fb3e5 --- /dev/null +++ b/internal/controller/mirror_creds_test.go @@ -0,0 +1,165 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "strings" + "testing" + + "github.com/go-logr/logr/funcr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +// Sentinel byte sequences that must never leak into logs or error strings. +const ( + sentinelCert = "SENTINEL-CERT-BYTES-b64f00" + sentinelKey = "SENTINEL-KEY-BYTES-deadbeef" + sentinelCA = "SENTINEL-CA-BYTES-cafe" + sentinelUsername = "SENTINEL-USERNAME-root" + sentinelPassword = "SENTINEL-PASSWORD-hunter2" +) + +func TestResolveSideCreds_NeverLogsValues(t *testing.T) { + tlsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "side-tls", Namespace: "default"}, + Data: map[string][]byte{ + "tls.crt": []byte(sentinelCert), + "tls.key": []byte(sentinelKey), + "ca.crt": []byte(sentinelCA), + }, + } + authSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "side-auth", Namespace: "default"}, + Data: map[string][]byte{ + "username": []byte(sentinelUsername), + "password": []byte(sentinelPassword), + }, + } + c := fake.NewClientBuilder().WithObjects(tlsSecret, authSecret).Build() + + var logLines []string + logger := funcr.New(func(prefix, args string) { + logLines = append(logLines, prefix+" "+args) + }, funcr.Options{Verbosity: 10}) + + ep := ecv1alpha1.EtcdMirrorEndpoint{ + TLS: &ecv1alpha1.EtcdMirrorTLS{SecretRef: &corev1.LocalObjectReference{Name: "side-tls"}}, + Auth: &ecv1alpha1.EtcdMirrorAuth{SecretRef: corev1.LocalObjectReference{Name: "side-auth"}}, + } + + layout, err := resolveSideCreds(t.Context(), c, logger, "default", "source", ep) + require.NoError(t, err) + assert.True(t, layout.HasClientCert) + assert.True(t, layout.HasCA) + assert.True(t, layout.HasAuth) + assert.NotEmpty(t, layout.AuthSecretRV) + + sentinels := []string{sentinelCert, sentinelKey, sentinelCA, sentinelUsername, sentinelPassword} + for _, line := range logLines { + for _, s := range sentinels { + assert.NotContains(t, line, s, "secret value leaked into a log line") + } + } + + t.Run("missing key errors name objects and keys, never data", func(t *testing.T) { + lopsided := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "lopsided-tls", Namespace: "default"}, + Data: map[string][]byte{"tls.crt": []byte(sentinelCert)}, + } + c := fake.NewClientBuilder().WithObjects(lopsided).Build() + ep := ecv1alpha1.EtcdMirrorEndpoint{ + TLS: &ecv1alpha1.EtcdMirrorTLS{SecretRef: &corev1.LocalObjectReference{Name: "lopsided-tls"}}, + } + _, err := resolveSideCreds(t.Context(), c, logger, "default", "source", ep) + require.Error(t, err) + var ce *credsError + require.ErrorAs(t, err, &ce) + assert.Equal(t, "InvalidTLSSecret", ce.Reason) + assert.NotContains(t, err.Error(), sentinelCert) + assert.Contains(t, err.Error(), "lopsided-tls") + }) + + t.Run("empty auth values are rejected without leaking", func(t *testing.T) { + emptyAuth := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "empty-auth", Namespace: "default"}, + Data: map[string][]byte{"username": []byte(sentinelUsername)}, + } + c := fake.NewClientBuilder().WithObjects(emptyAuth).Build() + ep := ecv1alpha1.EtcdMirrorEndpoint{ + Auth: &ecv1alpha1.EtcdMirrorAuth{SecretRef: corev1.LocalObjectReference{Name: "empty-auth"}}, + } + _, err := resolveSideCreds(t.Context(), c, logger, "default", "target", ep) + require.Error(t, err) + assert.NotContains(t, err.Error(), sentinelUsername) + }) + + t.Run("missing secret is SecretNotFound", func(t *testing.T) { + c := fake.NewClientBuilder().Build() + ep := ecv1alpha1.EtcdMirrorEndpoint{ + TLS: &ecv1alpha1.EtcdMirrorTLS{SecretRef: &corev1.LocalObjectReference{Name: "absent"}}, + } + _, err := resolveSideCreds(t.Context(), c, logger, "default", "source", ep) + var ce *credsError + require.ErrorAs(t, err, &ce) + assert.Equal(t, "SecretNotFound", ce.Reason) + }) +} + +func TestResolveSideCredsFinalizerTarget(t *testing.T) { + authSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "tgt-auth", Namespace: "default"}, + Data: map[string][]byte{ + "username": []byte(sentinelUsername + "\n"), + "password": []byte(sentinelPassword), + }, + } + c := fake.NewClientBuilder().WithObjects(authSecret).Build() + logger := funcr.New(func(string, string) {}, funcr.Options{}) + + em := minimalMirror() + em.Spec.Target.Prefix = "/mirrored/" + em.Spec.Target.Auth = &ecv1alpha1.EtcdMirrorAuth{SecretRef: corev1.LocalObjectReference{Name: "tgt-auth"}} + + tgt, err := resolveFinalizerTarget(t.Context(), c, logger, em) + require.NoError(t, err) + assert.Equal(t, []string{"tgt:2379"}, tgt.Endpoints) + assert.Nil(t, tgt.TLS) + assert.Equal(t, sentinelUsername, tgt.Username, "trailing newline must be stripped") + assert.Equal(t, sentinelPassword, tgt.Password) + assert.Equal(t, "/mirrored/\x00etcdmirror-checkpoint", tgt.Key) + + t.Run("explicit checkpoint key wins", func(t *testing.T) { + em := em.DeepCopy() + em.Spec.Checkpoint = &ecv1alpha1.EtcdMirrorCheckpointSpec{Key: "/mirrored/\x00custom"} + tgt, err := resolveFinalizerTarget(t.Context(), c, logger, em) + require.NoError(t, err) + assert.Equal(t, "/mirrored/\x00custom", tgt.Key) + }) +} + +// Guard against message drift: the etcdctl del command in the +// EmptyTargetViolation message must quote both range ends. +func TestEtcdctlDelRangeQuoting(t *testing.T) { + cmd := etcdctlDelCommand("/mirrored/") + assert.Equal(t, 2, strings.Count(cmd, `"/`)) +} diff --git a/pkg/mirroragent/agent.go b/pkg/mirroragent/agent.go new file mode 100644 index 00000000..380981ca --- /dev/null +++ b/pkg/mirroragent/agent.go @@ -0,0 +1,589 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "errors" + "fmt" + "maps" + "sync" + "sync/atomic" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +// errDrained signals a completed Drain cutover up the Run stack; Run maps it +// to a nil return. +var errDrained = errors.New("drain completed") + +// errFenceLost means a fence claim raced an older agent generation's write +// between our read and our takeover Txn. fenceViolation adopts the raced +// write's mod revision before returning it, so the caller's retry re-runs +// the takeover against the current fence: loadFence's loop re-reads and +// retries, and applyFenced retries any claim made before this generation's +// first successful commit (the genesis / startFromRevision claims on a +// fresh target). After that first commit (Agent.claimed), older generations +// fail their own compares and can never move the fence again, so a +// post-claim loss escalates to a permanent FenceError. +var errFenceLost = errors.New("fence claim raced an older generation write") + +// Agent is the replication engine. Create with New, drive with Run (once), +// observe with Snapshot from any goroutine. +type Agent struct { + cfg Config + src Client + dst Client + rw *rewriter + bo *backoff + + srcClusterID uint64 + dstClusterID uint64 + // trustProgress gates watermark advancement from watch progress + // notifications (source >= 3.4.25 / 3.5.8). + trustProgress bool + + // fence is the engine's cached copy of the reserved key's value; + // fenceModRev is the mod revision EVERY write path compares against. + // Both are owned by the Run goroutine. + fence FenceValue + fenceModRev int64 + // claimed is true once this generation committed its first fenced Txn: + // from then on a lost fence compare is a permanent violation, never a + // retryable claim race. Owned by the Run goroutine. + claimed bool + // prunePending mirrors FenceValue.PrunePending: set when a forced resync + // starts, stamped into every checkpoint, cleared only after the mandatory + // mark-and-sweep prune completed. Owned by the Run goroutine. + prunePending bool + + // watchCancel cancels the live source watch; applyFenced invokes it via + // cancelSourceWatch on sustained target backoff, and a long paced diff + // pass via maybeCancelWatchForPacedRepair, so clientv3's unbounded + // per-watcher response buffer cannot grow for the duration of a target + // stall or a paced repair. Owned by the Run goroutine (set by + // genesis/tail, consumed by the apply path, which runs on the same + // goroutine). + watchCancel func() + + drainReq atomic.Bool + + // consecutiveResyncs counts forced resyncs without reaching steady state + // in between (owned by the Run goroutine). restartBo paces Run's + // genesis-restart loop; it is deliberately separate from the shared bo, + // whose curves reset on every successful apply inside a scan attempt. + consecutiveResyncs int + restartBo *backoff + + // nextReconcile is the periodic pass's next deadline (zero until the + // first tail arms it); owned by the Run goroutine. Mandatory sweeps + // re-arm it — they produce the same signal. + nextReconcile time.Time + + mu sync.Mutex + snap Snapshot +} + +// startState says how a replication cycle begins: fresh genesis, resumed +// scan, resumed tail, or a forced resync. +type startState struct { + haveCheckpoint bool + scanning bool + scanCursor string + subRevision int64 + watermark int64 + // forced marks a forced resync: full re-scan plus a mandatory + // mark-and-sweep prune pass; RequireEmpty is NOT re-checked (the decoded, + // ownership-validated fence proves the destination data is this link's + // own). Persisted across restarts as FenceValue.PrunePending so a crash + // mid-forced-resync cannot silently drop the owed sweep. + forced bool + // rearmEmpty re-arms the RequireEmpty check (cluster-identity mismatch). + rearmEmpty bool +} + +// New validates cfg (after defaulting) and builds an Agent over the two +// clients. The caller owns client lifecycle and TLS/auth material. +func New(cfg Config, source, target Client) (*Agent, error) { + cfg = cfg.withDefaults() + if err := cfg.Validate(); err != nil { + return nil, err + } + return &Agent{ + cfg: cfg, + src: source, + dst: target, + rw: newRewriter(cfg), + bo: newBackoff(cfg.BackoffInitialDelay, cfg.BackoffMaxDelay), + restartBo: newBackoff(cfg.BackoffInitialDelay, cfg.BackoffMaxDelay), + snap: Snapshot{Phase: PhaseConnecting}, + }, nil +} + +// Snapshot returns a point-in-time copy of the agent's state, safe to retain. +func (a *Agent) Snapshot() Snapshot { + a.mu.Lock() + defer a.mu.Unlock() + s := a.snap + if a.snap.Cutover != nil { + c := *a.snap.Cutover + s.Cutover = &c + } + if a.snap.LastReconcileDrift != nil { + d := *a.snap.LastReconcileDrift + s.LastReconcileDrift = &d + } + s.ForcedResyncCountByReason = maps.Clone(a.snap.ForcedResyncCountByReason) + return s +} + +// RequestDrain flips a running Sync agent into a drain, as if Mode were +// ModeDrain. +func (a *Agent) RequestDrain() { a.drainReq.Store(true) } + +// Run executes the replication loop until ctx is cancelled (returns +// ctx.Err()), a Drain completes (returns nil), or a permanent failure occurs +// (returns the classified error). +func (a *Agent) Run(ctx context.Context) error { + if err := a.connect(ctx); err != nil { + return a.fail(ctx, err) + } + st, err := a.loadFence(ctx) + if err != nil { + return a.fail(ctx, err) + } + for { + err := a.cycle(ctx, st) + var restart *scanRestartError + switch { + case err == nil: + return nil // drain completed + case ctx.Err() != nil: + return ctx.Err() + case errors.As(err, &restart): + // Bounded genesis retry: restart the scan from a fresh R0. The + // dropped replay buffer may have held deletes, so the restarted + // attempt owes a mark-and-sweep (forced). The dedicated restart + // backoff keeps repeated restarts (churn outrunning the buffer) + // off a hot loop AND escalating: the shared curve resets on every + // successful apply inside a doomed attempt, so it would stay + // pinned at the initial delay forever. + a.noteScanRestart(restart) + if serr := sleepCtx(ctx, a.restartBo.next(ClassTransient)); serr != nil { + return serr + } + st = startState{forced: true} + case Classify(err) == ClassResync: + a.noteResync(err) + st = startState{forced: true} + default: + return a.fail(ctx, err) + } + } +} + +// cycle runs one replication attempt from the given start state. +func (a *Agent) cycle(ctx context.Context, st startState) error { + var err error + switch { + case st.haveCheckpoint && !st.scanning: + err = a.tail(ctx, nil, nil, st.watermark) + case !st.haveCheckpoint && !st.forced && a.cfg.StartRevision > 0: + err = a.startFromRevision(ctx) + default: + err = a.genesis(ctx, st) + } + if errors.Is(err, errDrained) { + return nil + } + return err +} + +// startFromRevision skips the genesis scan (fidelity-preserving snapshot +// seed) and tails from StartRevision+1. +func (a *Agent) startFromRevision(ctx context.Context) error { + if err := a.applyFenced(ctx, nil, a.newFence(a.cfg.StartRevision, false, "", 0), "", 0); err != nil { + return err + } + a.advanceWatermark(a.cfg.StartRevision) + return a.tail(ctx, nil, nil, a.cfg.StartRevision) +} + +// connect probes both sides' version and cluster identity (maintenance +// Status at connect) and enforces the >=3.4 hard floor. +func (a *Agent) connect(ctx context.Context) error { + a.setPhase(PhaseConnecting) + srcInfo, srcID, err := a.probe(ctx, "source", a.src) + if err != nil { + return err + } + dstInfo, dstID, err := a.probe(ctx, "target", a.dst) + if err != nil { + return err + } + a.srcClusterID, a.dstClusterID = srcID, dstID + a.trustProgress = srcInfo.TrustProgressNotify + a.update(func(s *Snapshot) { + s.SourceVersion = srcInfo.Version + s.TargetVersion = dstInfo.Version + s.SourceClusterID = srcID + s.TargetClusterID = dstID + }) + return nil +} + +func (a *Agent) probe(ctx context.Context, side string, cl Client) (versionInfo, uint64, error) { + eps := cl.Endpoints() + if len(eps) == 0 { + return versionInfo{}, 0, &ConfigError{Detail: side + " client has no endpoints"} + } + // Status dials the named endpoint directly, bypassing the balancer, so + // the probe must rotate endpoints itself: one blackholed member must not + // wedge the agent in Connecting while healthy quorum members exist. + attempt := 0 + for { + tctx, cancel := context.WithTimeout(ctx, a.cfg.RequestTimeout) + resp, err := cl.Status(tctx, eps[attempt%len(eps)]) + cancel() + attempt++ + if err == nil { + if attempt > 1 { + a.bo.noteSuccess() + } + vi, verr := classifyVersion(side, resp.Version) + if verr != nil { + return versionInfo{}, 0, verr + } + learner := resp.IsLearner + a.update(func(s *Snapshot) { + if side == "source" { + s.SourceLearner = learner + } else { + s.TargetLearner = learner + } + }) + return vi, resp.Header.ClusterId, nil + } + if ctx.Err() != nil { + return versionInfo{}, 0, ctx.Err() + } + class := Classify(err) + if class == ClassPermanent || class == ClassResync { + return versionInfo{}, 0, fmt.Errorf("probing %s: %w", side, err) + } + a.recordErr(err, class) + if serr := sleepCtx(ctx, a.bo.next(class)); serr != nil { + return versionInfo{}, 0, serr + } + } +} + +// loadFence reads the reserved key, validates ownership, and — when a valid +// checkpoint of this link exists — takes the fence over for this agent +// generation so any straggler generation fails its next compare. +func (a *Agent) loadFence(ctx context.Context) (startState, error) { + for { + resp, err := a.getRetry(ctx, a.dst, a.cfg.CheckpointKey) + if err != nil { + return startState{}, err + } + if len(resp.Kvs) == 0 { + // Fresh target: the fence is claimed at genesis start, after the + // RequireEmpty gate, so a violation writes nothing. + a.fenceModRev = 0 + return startState{}, nil + } + kv := resp.Kvs[0] + a.fenceModRev = kv.ModRevision + f, derr := DecodeFenceValue(kv.Value) + if derr != nil { + // Corrupt or unknown-version checkpoint: fail CLOSED. Nothing + // about the stored value is knowable — not the owning link, not + // the epoch, not whether a cutover already flipped the role to + // Primary — so no write (least of all a resync's prune) is + // provably safe. Permanent: the operator must inspect the + // reserved key and delete it to recover. + return startState{}, fmt.Errorf("reserved key %q: %w", a.cfg.CheckpointKey, derr) + } + takeover, st, verr := a.validateFence(f) + if verr != nil { + return startState{}, verr + } + if !takeover { + return st, nil + } + // Take the fence over for this generation before anything else runs. + f.Epoch = a.cfg.Epoch + err = a.commitFenced(ctx, nil, f) + if err == nil { + a.advanceWatermark(f.Watermark) + return startState{ + haveCheckpoint: true, + scanning: f.Scanning, + scanCursor: f.ScanCursor, + subRevision: f.SubRevision, + watermark: f.Watermark, + // A crash mid-forced-resync leaves the owed mark-and-sweep + // recorded in the fence; the resumed scan must still prune. + forced: f.PrunePending, + }, nil + } + if errors.Is(err, errFenceLost) { + continue // an older generation wrote in the read/claim window + } + class := Classify(err) + switch class { + case ClassPermanent, ClassResync: + return startState{}, err + case ClassQuota: + a.recordErr(err, class) + a.update(func(s *Snapshot) { s.QuotaExhausted = true; s.Phase = PhaseDegraded }) + if serr := sleepCtx(ctx, a.cfg.QuotaProbeInterval); serr != nil { + return startState{}, serr + } + default: + a.recordErr(err, class) + if serr := sleepCtx(ctx, a.bo.next(class)); serr != nil { + return startState{}, serr + } + } + } +} + +// validateFence checks a decoded checkpoint against this agent's identity. +// takeover is true when the fence is ours to take over; otherwise st is the +// forced-resync start state or err is terminal. +func (a *Agent) validateFence(f FenceValue) (takeover bool, st startState, err error) { + if f.LinkUID != a.cfg.LinkUID { + return false, startState{}, &FenceError{Detail: fmt.Sprintf( + "reserved key %q is owned by link %q, not %q", + a.cfg.CheckpointKey, f.LinkUID, a.cfg.LinkUID)} + } + if f.Role == RolePrimary { + return false, startState{}, &FenceError{ + Detail: "fence role is Primary: cutover completed, mirror writes are forbidden", + } + } + if f.Epoch > a.cfg.Epoch { + return false, startState{}, &FenceError{Detail: fmt.Sprintf( + "newer agent epoch %d owns the link (this agent is epoch %d)", + f.Epoch, a.cfg.Epoch)} + } + if f.SourceClusterID != a.srcClusterID || f.TargetClusterID != a.dstClusterID { + a.noteResync(&ResyncError{Reason: ResyncReasonClusterIDMismatch, Cause: fmt.Errorf( + "checkpoint bound to source=%d target=%d, probed source=%d target=%d", + f.SourceClusterID, f.TargetClusterID, a.srcClusterID, a.dstClusterID)}) + return false, startState{forced: true, rearmEmpty: true}, nil + } + return true, startState{}, nil +} + +// newFence builds the checkpoint document for this agent generation. +func (a *Agent) newFence(watermark int64, scanning bool, cursor string, subrev int64) FenceValue { + return FenceValue{ + LinkUID: a.cfg.LinkUID, + Epoch: a.cfg.Epoch, + Role: RoleMirror, + Watermark: watermark, + Scanning: scanning, + ScanCursor: cursor, + SubRevision: subrev, + PrunePending: a.prunePending, + SourceClusterID: a.srcClusterID, + TargetClusterID: a.dstClusterID, + } +} + +// noteResync records a forced resync and drives the livelock detector. +func (a *Agent) noteResync(err error) { + reason := ResyncReasonCompacted + var re *ResyncError + if errors.As(err, &re) { + reason = re.Reason + } + a.consecutiveResyncs++ + loop := a.consecutiveResyncs >= a.cfg.ResyncLoopThreshold + a.update(func(s *Snapshot) { + s.ForcedResyncCount++ + if s.ForcedResyncCountByReason == nil { + s.ForcedResyncCountByReason = make(map[ResyncReason]int64, 2) + } + s.ForcedResyncCountByReason[reason]++ + s.LastResyncReason = reason + s.Compacted = reason == ResyncReasonCompacted + if loop { + s.ResyncLoopDetected = true + } + }) +} + +// noteScanRestart records an aborted genesis attempt (buffer overflow or a +// watch reconnect below the compact revision mid-scan). Restarts do not +// invalidate the checkpoint — ForcedResyncCount is untouched — but they +// count toward the same livelock detector: repeated restarts are the +// signature of churn or retention outrunning scan throughput. +func (a *Agent) noteScanRestart(e *scanRestartError) { + a.consecutiveResyncs++ + loop := a.consecutiveResyncs >= a.cfg.ResyncLoopThreshold + a.update(func(s *Snapshot) { + s.ScanRestartCount++ + s.LastScanRestartCause = e.Cause + if loop { + s.ResyncLoopDetected = true + } + }) +} + +// steadyState is reached on the first successfully applied LIVE watch +// response of a tail: it resets the resync-loop detector and the restart +// backoff. Genesis replay-buffer applies must never reach here — they run +// INSIDE the resync the detector is counting, and a churning source (the +// canonical livelock trigger) guarantees a non-empty replay buffer, so a +// replay-driven reset would keep the detector at zero forever. +func (a *Agent) steadyState() { + a.restartBo.reset() + if a.consecutiveResyncs == 0 { + return + } + a.consecutiveResyncs = 0 + a.update(func(s *Snapshot) { + s.ResyncLoopDetected = false + s.Compacted = false + }) +} + +// cancelSourceWatch tears down the live source watch (if any) so clientv3 +// stops buffering undelivered responses while the target is parked or in +// sustained backoff; the tail re-watches from the checkpoint watermark once +// applies succeed again. Idempotent. +func (a *Agent) cancelSourceWatch() { + if a.watchCancel != nil { + a.watchCancel() + a.watchCancel = nil + } +} + +func (a *Agent) fail(ctx context.Context, err error) error { + if ctx.Err() != nil { + return ctx.Err() + } + class := Classify(err) + reason := ReasonFor(err) + a.update(func(s *Snapshot) { + s.LastError = err.Error() + s.LastErrorClass = class + s.LastErrorReason = reason + if s.Phase != PhaseDrained { + s.Phase = PhaseFailed + } + }) + return err +} + +func (a *Agent) update(fn func(*Snapshot)) { + a.mu.Lock() + defer a.mu.Unlock() + fn(&a.snap) +} + +func (a *Agent) setPhase(p Phase) { + a.update(func(s *Snapshot) { s.Phase = p }) +} + +func (a *Agent) phase() Phase { + a.mu.Lock() + defer a.mu.Unlock() + return a.snap.Phase +} + +func (a *Agent) watermark() int64 { + a.mu.Lock() + defer a.mu.Unlock() + return a.snap.Watermark +} + +func (a *Agent) advanceWatermark(rev int64) { + a.update(func(s *Snapshot) { + if rev > s.Watermark { + s.Watermark = rev + } + s.LastProgressTime = time.Now() + }) +} + +func (a *Agent) recordErr(err error, class Class) { + reason := ReasonFor(err) + a.update(func(s *Snapshot) { + s.LastError = err.Error() + s.LastErrorClass = class + s.LastErrorReason = reason + }) +} + +// pace enforces MaxOpsPerSecond with simple pre-write sleeping. +func (a *Agent) pace(ctx context.Context, n int) { + if a.cfg.MaxOpsPerSecond <= 0 || n == 0 { + return + } + d := time.Duration(n) * time.Second / time.Duration(a.cfg.MaxOpsPerSecond) + _ = sleepCtx(ctx, d) +} + +func sleepCtx(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} + +// getRetry performs a unary Get under the per-RPC deadline with the standard +// read retry policy: transient/throttle back off, resync and permanent +// propagate. A success after retries resets the backoff curves, so one old +// saturated burst does not pin every later isolated retry at the max delay. +func (a *Agent) getRetry( + ctx context.Context, cl Client, key string, opts ...clientv3.OpOption, +) (*clientv3.GetResponse, error) { + retried := false + for { + tctx, cancel := context.WithTimeout(ctx, a.cfg.RequestTimeout) + resp, err := cl.Get(tctx, key, opts...) + cancel() + if err == nil { + if retried { + a.bo.noteSuccess() + } + return resp, nil + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + class := Classify(err) + if class == ClassPermanent || class == ClassResync { + return nil, err + } + a.recordErr(err, class) + retried = true + if serr := sleepCtx(ctx, a.bo.next(class)); serr != nil { + return nil, serr + } + } +} diff --git a/pkg/mirroragent/agent_test.go b/pkg/mirroragent/agent_test.go new file mode 100644 index 00000000..dbad35bc --- /dev/null +++ b/pkg/mirroragent/agent_test.go @@ -0,0 +1,68 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd/api/v3/etcdserverpb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// statusStubClient stubs only the maintenance Status probe (and Endpoints); +// the embedded Client is never touched by the probe path. +type statusStubClient struct { + Client + learner bool +} + +func (c *statusStubClient) Endpoints() []string { return []string{"stub:2379"} } + +func (c *statusStubClient) Status(_ context.Context, _ string) (*clientv3.StatusResponse, error) { + return &clientv3.StatusResponse{ + Header: &etcdserverpb.ResponseHeader{ClusterId: 42}, + Version: "3.5.9", + IsLearner: c.learner, + }, nil +} + +// TestProbeRecordsLearner: the connect-time Status probe must record +// IsLearner into the snapshot for its side — the LearnerEndpoint condition's +// only input. +func TestProbeRecordsLearner(t *testing.T) { + a := newTestAgent(t) + + _, id, err := a.probe(t.Context(), "source", &statusStubClient{learner: true}) + require.NoError(t, err) + require.EqualValues(t, 42, id) + snap := a.Snapshot() + require.True(t, snap.SourceLearner) + require.False(t, snap.TargetLearner, "the target side must be untouched by a source probe") + + _, _, err = a.probe(t.Context(), "target", &statusStubClient{learner: false}) + require.NoError(t, err) + snap = a.Snapshot() + require.True(t, snap.SourceLearner) + require.False(t, snap.TargetLearner) + + _, _, err = a.probe(t.Context(), "target", &statusStubClient{learner: true}) + require.NoError(t, err) + require.True(t, a.Snapshot().TargetLearner) +} diff --git a/pkg/mirroragent/apply.go b/pkg/mirroragent/apply.go new file mode 100644 index 00000000..35145f5c --- /dev/null +++ b/pkg/mirroragent/apply.go @@ -0,0 +1,286 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "errors" + "fmt" + "strings" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// backoffRoundsBeforeWatchCancel is how many transient/throttle backoff +// rounds a single fenced apply endures before the live source watch is +// cancelled: clientv3 buffers undelivered watch responses without bound, so +// a sustained target stall must stop the source stream (the tail re-watches +// from the checkpoint watermark once applies succeed again). Quota parks +// cancel immediately — they are expected to last minutes to hours. +const backoffRoundsBeforeWatchCancel = 3 + +// applyOps writes one flush set plus checkpoint f in a single fenced Txn, +// through the class-appropriate retry policy. A multi-revision set the +// target rejects for its Txn limits gets ONE shrink attempt at revision +// granularity (the checkpoint advancing with each sub-Txn); a single +// revision that alone trips the limits is irreducible and stays permanent. +func (a *Agent) applyOps(ctx context.Context, fs *flushSet, f FenceValue) error { + a.pace(ctx, len(fs.ops)) + err := a.applyFlushSet(ctx, fs, f) + var tle *TooLargeError + if err == nil || !errors.As(err, &tle) || len(fs.groups) <= 1 { + return err + } + // Shrink: the whole set fit the engine's own watermarks but not the + // target's --max-txn-ops / --max-request-bytes (foreign cluster; its + // flags are not inspectable). Re-commit revision by revision — every + // sub-Txn still cuts at a revision boundary. + for _, g := range fs.groups { + sub := flushSet{ + ops: g.ops, + groups: []revGroup{g}, + watermark: g.rev, + lastSrcKey: g.ops[len(g.ops)-1].srcKey, + } + fsub := f + fsub.Watermark = g.rev + if fsub.Scanning { + fsub.ScanCursor = sub.lastSrcKey + } + if serr := a.applyFlushSet(ctx, &sub, fsub); serr != nil { + return serr + } + } + return nil +} + +// applyFlushSet converts one flush set to ops and commits it fenced. +func (a *Agent) applyFlushSet(ctx context.Context, fs *flushSet, f FenceValue) error { + ops := make([]clientv3.Op, 0, len(fs.ops)) + var nBytes int64 + for _, o := range fs.ops { + nBytes += o.bytes() + if o.isDelete { + ops = append(ops, clientv3.OpDelete(o.key)) + } else { + ops = append(ops, clientv3.OpPut(o.key, o.value)) + } + } + return a.applyFenced(ctx, ops, f, fs.ops[0].key, nBytes) +} + +// noteApplySuccess records one successful fenced commit: counters advance and +// every last-error field clears — including LastErrorReason, whose wire +// contract is "" once the last attempt succeeded. +func (a *Agent) noteApplySuccess(nOps int, prior Phase) { + a.bo.noteSuccess() + a.update(func(s *Snapshot) { + s.KeysAppliedTotal += int64(nOps) + s.Throttled = false + s.QuotaExhausted = false + s.LastError = "" + s.LastErrorClass = "" + s.LastErrorReason = "" + if s.Phase == PhaseDegraded && prior != PhaseDegraded { + s.Phase = prior + } + }) +} + +// applyFenced commits ops plus the checkpoint write under the fence compare, +// retrying per class: transient and throttle back off on their own curves, +// quota parks on the flat probe interval (never backoff — quota only heals +// when an operator acts), resync and permanent errors propagate. A fence +// claim (before this generation's first successful commit) that loses a race +// against an older generation's last write retries the takeover; post-claim, +// a moved fence is a loud permanent failure. +func (a *Agent) applyFenced( + ctx context.Context, ops []clientv3.Op, f FenceValue, firstKey string, nBytes int64, +) error { + prior := a.phase() + rounds := 0 + for { + err := a.commitFenced(ctx, ops, f) + if err == nil { + a.noteApplySuccess(len(ops), prior) + return nil + } + if ctx.Err() != nil { + return ctx.Err() + } + if errors.Is(err, errFenceLost) { + if !a.claimed { + // Pre-claim race: an older generation wrote between our fence + // read and this claim. fenceViolation adopted the raced mod + // revision, so the retry re-runs the takeover against it. + a.recordErr(err, ClassTransient) + if serr := sleepCtx(ctx, a.bo.next(ClassTransient)); serr != nil { + return serr + } + continue + } + // Post-claim, older generations fail their own compares and can + // never move the fence again: surface loudly rather than retrying + // against a stale mod revision forever. + err = &FenceError{Detail: "checkpoint mod revision moved under an active generation"} + } + class := Classify(err) + if class == ClassPermanent && isOversized(err) { + err = &TooLargeError{ + Key: RedactKey(a.cfg.EffectiveDestPrefix(), []byte(firstKey)), + Ops: len(ops) + 1, + Bytes: nBytes, + Cause: err, + } + } + a.recordErr(err, class) + switch class { + case ClassQuota: + a.cancelSourceWatch() + a.update(func(s *Snapshot) { + s.QuotaExhausted = true + s.Phase = PhaseDegraded + }) + if serr := sleepCtx(ctx, a.cfg.QuotaProbeInterval); serr != nil { + return serr + } + case ClassThrottle: + if rounds++; rounds >= backoffRoundsBeforeWatchCancel { + a.cancelSourceWatch() + } + a.update(func(s *Snapshot) { + s.Throttled = true + s.Phase = PhaseDegraded + }) + if serr := sleepCtx(ctx, a.bo.next(class)); serr != nil { + return serr + } + case ClassTransient: + if rounds++; rounds >= backoffRoundsBeforeWatchCancel { + a.cancelSourceWatch() + } + a.setPhase(PhaseDegraded) + if serr := sleepCtx(ctx, a.bo.next(class)); serr != nil { + return serr + } + default: + return err + } + } +} + +// commitFenced is one fenced Txn attempt: ops plus the checkpoint write in +// the reserved op slot, guarded by the mod-revision compare on the reserved +// key. On success the cached fence and mod revision advance together. A +// failed compare is resolved by fenceViolation, which recognizes this +// generation's own ambiguously-timed-out commit and adopts it as success. +func (a *Agent) commitFenced(ctx context.Context, ops []clientv3.Op, f FenceValue) error { + val, err := f.Encode() + if err != nil { + // Encoding our own fence value can only fail on an engine bug; fail + // closed the same way a corrupt stored checkpoint does. + return &CheckpointInvalidError{Reason: fmt.Sprintf("encoding checkpoint: %v", err)} + } + cmp := clientv3.Compare(clientv3.ModRevision(a.cfg.CheckpointKey), "=", a.fenceModRev) + all := make([]clientv3.Op, 0, len(ops)+1) + all = append(all, ops...) + all = append(all, clientv3.OpPut(a.cfg.CheckpointKey, val)) + tctx, cancel := context.WithTimeout(ctx, a.cfg.RequestTimeout) + resp, err := a.dst.Txn(tctx).If(cmp).Then(all...).Commit() + cancel() + if err != nil { + return err + } + if !resp.Succeeded { + return a.fenceViolation(ctx, f, val) + } + a.fence = f + a.fenceModRev = resp.Header.Revision + a.claimed = true + return nil +} + +// fenceViolation resolves a failed fence compare by re-reading the reserved +// key — never a blind re-Commit (doc.go's retry-ownership contract). Three +// outcomes: +// +// - The stored value is byte-identical to the value this attempt was +// writing: an earlier attempt of this exact Txn committed but its +// response was lost (the classic ambiguous timeout — the Txn's own +// checkpoint Put bumped the fence ModRevision). The data ops landed +// exactly once; adopt the new mod revision and report success (nil). +// - The stored fence is an OLDER generation of this link: a claim raced +// the old generation's last write. Adopt the raced mod revision and +// return errFenceLost so the caller retries the takeover. +// - Anything else (another link, a newer epoch, a Primary role we did not +// write): a genuine, permanent fence violation. +func (a *Agent) fenceViolation(ctx context.Context, f FenceValue, attempted string) error { + resp, err := a.getRetry(ctx, a.dst, a.cfg.CheckpointKey) + if err != nil { + // getRetry already retried transient/throttle reads; what escapes is + // cancellation or a permanent read failure. + return fmt.Errorf("re-reading fence after a failed compare: %w", err) + } + if len(resp.Kvs) == 0 { + return &FenceError{Detail: "the reserved key was deleted under an active generation"} + } + kv := resp.Kvs[0] + if string(kv.Value) == attempted { + a.fence = f + a.fenceModRev = kv.ModRevision + a.claimed = true + return nil + } + stored, derr := DecodeFenceValue(kv.Value) + if derr != nil { + return &FenceError{Detail: "checkpoint mod revision moved and the current fence is undecodable"} + } + switch { + case stored.LinkUID != a.cfg.LinkUID: + return &FenceError{Detail: fmt.Sprintf("fence taken over by link %q", stored.LinkUID)} + case stored.Role == RolePrimary: + return &FenceError{ + Detail: "fence role is Primary: cutover completed, mirror writes are forbidden", + } + case stored.Epoch > a.cfg.Epoch: + return &FenceError{Detail: fmt.Sprintf( + "newer agent epoch %d owns the link (this agent is epoch %d)", stored.Epoch, a.cfg.Epoch)} + case stored.Epoch < a.cfg.Epoch: + a.fenceModRev = kv.ModRevision + return errFenceLost + default: + return &FenceError{Detail: "another agent with the same epoch holds the fence"} + } +} + +// isOversized reports whether err is the server's Txn size/op-count limit or +// the gRPC client send cap — permanent errors that must surface the poison +// batch, never be retried as throttling. +func isOversized(err error) bool { + switch rpctypes.Error(err) { + case rpctypes.ErrTooManyOps, rpctypes.ErrRequestTooLarge: + return true + } + if s, ok := status.FromError(err); ok { + return s.Code() == codes.ResourceExhausted && strings.Contains(s.Message(), "larger than max") + } + return false +} diff --git a/pkg/mirroragent/backoff.go b/pkg/mirroragent/backoff.go new file mode 100644 index 00000000..4f72512a --- /dev/null +++ b/pkg/mirroragent/backoff.go @@ -0,0 +1,90 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import "time" + +// backoff produces class-specific retry delays. Connection-class errors get +// a standard exponential curve within [initial, max]; throttling-class +// errors get a more conservative curve derived from the same bounds (start +// 4x higher, cap 2x higher) — the target asked us to slow down, so we slow +// down harder and longer. Quota and permanent classes never route here. +// +// Retry ownership: clientv3 auto-retries Get only on codes.Unavailable; +// Txn/Put/Delete are write-at-most-once (client-retried only when no +// connection was ever established). The engine owns 100% of write-path +// retry/backoff, driven by [Classify] and these curves. A fenced-Txn retry +// after an ambiguous timeout must re-read the fence first — the Txn's own +// success bumped the fence ModRevision. +type backoff struct { + initial time.Duration + max time.Duration + + transientNext time.Duration + throttleNext time.Duration + lastThrottle time.Time +} + +func newBackoff(initial, maxDelay time.Duration) *backoff { + return &backoff{initial: initial, max: maxDelay} +} + +// next returns the delay before the next attempt for the given class and +// advances that class's curve. Classes without a backoff policy (resync, +// quota, permanent) fall back to the transient curve — callers are expected +// to handle them before consulting backoff. +func (b *backoff) next(c Class) time.Duration { + if c == ClassThrottle { + b.lastThrottle = time.Now() + if b.throttleNext == 0 { + b.throttleNext = minDuration(4*b.initial, 2*b.max) + } + d := b.throttleNext + b.throttleNext = minDuration(2*b.throttleNext, 2*b.max) + return d + } + if b.transientNext == 0 { + b.transientNext = b.initial + } + d := b.transientNext + b.transientNext = minDuration(2*b.transientNext, b.max) + return d +} + +// reset clears both curves unconditionally. +func (b *backoff) reset() { + b.transientNext = 0 + b.throttleNext = 0 +} + +// noteSuccess resets the transient curve immediately but the throttle curve +// only after a full max-delay interval without throttle errors: a target +// still intermittently rejecting the write rate must keep escalating instead +// of restarting from the floor after every successful batch. +func (b *backoff) noteSuccess() { + b.transientNext = 0 + if b.throttleNext != 0 && time.Since(b.lastThrottle) >= b.max { + b.throttleNext = 0 + } +} + +func minDuration(a, b time.Duration) time.Duration { + if a < b { + return a + } + return b +} diff --git a/pkg/mirroragent/backoff_test.go b/pkg/mirroragent/backoff_test.go new file mode 100644 index 00000000..46cf7906 --- /dev/null +++ b/pkg/mirroragent/backoff_test.go @@ -0,0 +1,96 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestBackoffTransientCurve(t *testing.T) { + b := newBackoff(time.Second, 8*time.Second) + assert.Equal(t, 1*time.Second, b.next(ClassTransient)) + assert.Equal(t, 2*time.Second, b.next(ClassTransient)) + assert.Equal(t, 4*time.Second, b.next(ClassTransient)) + assert.Equal(t, 8*time.Second, b.next(ClassTransient)) + assert.Equal(t, 8*time.Second, b.next(ClassTransient), "the transient curve caps at max") +} + +// TestBackoffThrottleCurveDistinct: the target asked us to slow down, so the +// throttle curve starts higher (4x) and caps higher (2x) than the transient +// curve — and the two advance independently. +func TestBackoffThrottleCurveDistinct(t *testing.T) { + b := newBackoff(time.Second, 8*time.Second) + assert.Equal(t, 4*time.Second, b.next(ClassThrottle)) + assert.Equal(t, 8*time.Second, b.next(ClassThrottle)) + assert.Equal(t, 16*time.Second, b.next(ClassThrottle)) + assert.Equal(t, 16*time.Second, b.next(ClassThrottle), "the throttle curve caps at 2x max") + + assert.Equal(t, 1*time.Second, b.next(ClassTransient), + "throttle advancement must not move the transient curve") +} + +func TestBackoffThrottleStartCappedForTightBounds(t *testing.T) { + b := newBackoff(time.Second, time.Second) + assert.Equal(t, 2*time.Second, b.next(ClassThrottle), + "4x initial is capped at 2x max when the bounds are tight") +} + +func TestBackoffReset(t *testing.T) { + b := newBackoff(time.Second, 8*time.Second) + _ = b.next(ClassTransient) + _ = b.next(ClassTransient) + _ = b.next(ClassThrottle) + b.reset() + assert.Equal(t, 1*time.Second, b.next(ClassTransient), "reset restarts the transient curve") + assert.Equal(t, 4*time.Second, b.next(ClassThrottle), "reset restarts the throttle curve") +} + +// TestBackoffNoteSuccessPreservesThrottleCurve: a success right after a +// throttle delay resets only the transient curve — the throttle curve keeps +// escalating across intermittent successes and resets only after a full +// max-delay interval without throttle errors. +func TestBackoffNoteSuccessPreservesThrottleCurve(t *testing.T) { + b := newBackoff(time.Millisecond, 20*time.Millisecond) + _ = b.next(ClassTransient) + _ = b.next(ClassTransient) + first := b.next(ClassThrottle) + b.noteSuccess() + assert.Equal(t, time.Millisecond, b.next(ClassTransient), + "noteSuccess restarts the transient curve immediately") + assert.Greater(t, b.next(ClassThrottle), first, + "the throttle curve must keep escalating across an immediate success") + + time.Sleep(25 * time.Millisecond) // > max: a genuinely healthy stretch + b.noteSuccess() + assert.Equal(t, 4*time.Millisecond, b.next(ClassThrottle), + "a max-delay-long throttle-free stretch restarts the throttle curve") +} + +// TestBackoffNonRetryClassesUseTransientCurve documents the contract that +// resync/quota/permanent never legitimately reach backoff: callers handle +// them first (quota parks on the flat QuotaProbeInterval instead — pinned by +// the TestTargetQuotaExhausted integration test). If one slips through, it +// falls back to the standard curve rather than spinning. +func TestBackoffNonRetryClassesUseTransientCurve(t *testing.T) { + b := newBackoff(time.Second, 8*time.Second) + assert.Equal(t, 1*time.Second, b.next(ClassQuota)) + assert.Equal(t, 2*time.Second, b.next(ClassPermanent)) + assert.Equal(t, 4*time.Second, b.next(ClassResync)) +} diff --git a/pkg/mirroragent/batch.go b/pkg/mirroragent/batch.go new file mode 100644 index 00000000..d31be421 --- /dev/null +++ b/pkg/mirroragent/batch.go @@ -0,0 +1,187 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +// kvOp is one target-side write. Keys are already rewritten; srcKey is +// retained for scan-cursor bookkeeping only. +type kvOp struct { + key string + value string + isDelete bool + srcKey string +} + +func (o kvOp) bytes() int64 { return int64(len(o.key) + len(o.value)) } + +// revGroup is the complete set of in-scope ops of ONE source revision — the +// atom the batcher never splits. Scan pages use synthetic groups (one key +// per group, rev = the scan base) since a snapshot has no per-key revision +// boundaries to preserve. +type revGroup struct { + rev int64 + ops []kvOp +} + +func (g revGroup) bytes() int64 { + var n int64 + for _, o := range g.ops { + n += o.bytes() + } + return n +} + +// flushSet is one target Txn's worth of ops plus the checkpoint metadata +// that rides in the same Txn's reserved op slot. +type flushSet struct { + ops []kvOp + // groups preserves the revision boundaries inside ops, so a set the + // target rejects for its Txn limits can be re-committed at revision + // granularity (one shrink attempt) instead of failing permanently. + groups []revGroup + // watermark is the last complete source revision in the set (or the + // scan base for scan flushes). + watermark int64 + // lastSrcKey is the last source key in the set, for the scan cursor. + lastSrcKey string + // oversized marks a set that alone exceeds maxOps or maxBytes: a single + // source revision applied as one oversized Txn (the checkpoint is held + // until it lands — it rides in the same Txn). If the target rejects it, + // the set is irreducible: the error is permanent and the offending key + // is surfaced. + oversized bool +} + +// batcher coalesces whole revision groups into flush sets, flushing ONLY at +// source-revision boundaries. maxOps already has the checkpoint's reserved +// op slot subtracted (MaxTxnOps - 1); maxBytes is the TxnFlushBytes +// watermark. +type batcher struct { + maxOps int + maxBytes int64 + + pending []revGroup + pendingKeys map[string]struct{} + pendingOps int + pendingBytes int64 +} + +func newBatcher(maxTxnOps int, txnFlushBytes int64) *batcher { + return &batcher{ + // One op slot is always reserved for the checkpoint write. + maxOps: maxTxnOps - 1, + maxBytes: txnFlushBytes, + } +} + +// add appends one whole revision group and returns any flush sets that +// became due. A group is never split: if appending it would overflow the +// pending set, the pending set is flushed first; a group that alone exceeds +// the limits becomes its own oversized flush set. +func (b *batcher) add(g revGroup) []flushSet { + if len(g.ops) == 0 { + return nil + } + var out []flushSet + gBytes := g.bytes() + + // Flush what's pending if this group doesn't fit on top of it, or if it + // touches a key already pending: etcd rejects duplicate keys within one + // Txn (a catch-up watch response batches up to 1000 revisions, so one + // key modified twice in the window would otherwise put two ops on the + // same key into one flush set — a deterministic permanent failure). + if len(b.pending) > 0 && + (b.pendingOps+len(g.ops) > b.maxOps || b.pendingBytes+gBytes > b.maxBytes || + b.overlapsPending(g)) { + if fs := b.flush(); fs != nil { + out = append(out, *fs) + } + } + + b.pending = append(b.pending, g) + if b.pendingKeys == nil { + b.pendingKeys = make(map[string]struct{}, len(g.ops)) + } + for _, op := range g.ops { + b.pendingKeys[op.key] = struct{}{} + } + b.pendingOps += len(g.ops) + b.pendingBytes += gBytes + + // Flush immediately once the watermarks are reached — including the + // oversized single-group case. + if b.pendingOps >= b.maxOps || b.pendingBytes >= b.maxBytes { + if fs := b.flush(); fs != nil { + out = append(out, *fs) + } + } + return out +} + +// flush drains whatever is pending into one flush set (nil when empty). +// Called by add at watermarks and by the apply loop at the end of each +// watch response / scan page so writes are never held waiting for more +// input. +func (b *batcher) flush() *flushSet { + if len(b.pending) == 0 { + return nil + } + fs := flushSet{ + groups: b.pending, + watermark: b.pending[len(b.pending)-1].rev, + oversized: b.pendingOps > b.maxOps || b.pendingBytes > b.maxBytes, + } + fs.ops = make([]kvOp, 0, b.pendingOps) + for _, g := range b.pending { + fs.ops = append(fs.ops, g.ops...) + } + fs.lastSrcKey = fs.ops[len(fs.ops)-1].srcKey + b.pending = nil + b.pendingKeys = nil + b.pendingOps = 0 + b.pendingBytes = 0 + return &fs +} + +// overlapsPending reports whether any of g's keys is already pending. +func (b *batcher) overlapsPending(g revGroup) bool { + for _, op := range g.ops { + if _, ok := b.pendingKeys[op.key]; ok { + return true + } + } + return false +} + +// groupByRevision converts an ordered event stream (already rewritten and +// filtered) into revision groups, preserving order. Events of one revision +// are always contiguous in an etcd watch stream. +func groupByRevision(ops []kvOp, revs []int64) []revGroup { + if len(ops) != len(revs) || len(ops) == 0 { + return nil + } + var groups []revGroup + cur := revGroup{rev: revs[0]} + for i, op := range ops { + if revs[i] != cur.rev { + groups = append(groups, cur) + cur = revGroup{rev: revs[i]} + } + cur.ops = append(cur.ops, op) + } + groups = append(groups, cur) + return groups +} diff --git a/pkg/mirroragent/batch_test.go b/pkg/mirroragent/batch_test.go new file mode 100644 index 00000000..0cefbad7 --- /dev/null +++ b/pkg/mirroragent/batch_test.go @@ -0,0 +1,192 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mkGroup builds a revGroup of n one-byte-key/valueLen-byte-value ops. +func mkGroup(rev int64, n, valueLen int) revGroup { + g := revGroup{rev: rev} + for i := range n { + g.ops = append(g.ops, kvOp{ + key: fmt.Sprintf("k%d-%d", rev, i), + value: string(make([]byte, valueLen)), + srcKey: fmt.Sprintf("s%d-%d", rev, i), + }) + } + return g +} + +// TestBatcherReservesCheckpointSlot pins the MaxTxnOps-1 invariant: with +// MaxTxnOps=5 the batcher flushes at exactly 4 data ops — one op slot is +// always reserved for the checkpoint write riding the same Txn. +func TestBatcherReservesCheckpointSlot(t *testing.T) { + b := newBatcher(5, 1<<20) + var flushed []flushSet + for rev := int64(1); rev <= 3; rev++ { + flushed = append(flushed, b.add(mkGroup(rev, 1, 1))...) + } + require.Empty(t, flushed, "3 ops must still be pending below the 4-op watermark") + + flushed = b.add(mkGroup(4, 1, 1)) + require.Len(t, flushed, 1, "the 4th op hits MaxTxnOps-1 exactly and must flush") + assert.Len(t, flushed[0].ops, 4) + assert.False(t, flushed[0].oversized) + assert.EqualValues(t, 4, flushed[0].watermark, "watermark is the last complete revision") + assert.Nil(t, b.flush(), "nothing may remain pending after the watermark flush") +} + +// TestBatcherNeverSplitsRevision: a revision group that does not fit on top +// of the pending set flushes the pending set first and stays whole. +func TestBatcherNeverSplitsRevision(t *testing.T) { + b := newBatcher(6, 1<<20) // 5 data op slots + first := b.add(mkGroup(10, 3, 1)) + require.Empty(t, first) + + // 3 pending + 3 incoming > 5: pending flushes alone, the new group pends. + flushed := b.add(mkGroup(11, 3, 1)) + require.Len(t, flushed, 1) + assert.Len(t, flushed[0].ops, 3) + assert.EqualValues(t, 10, flushed[0].watermark) + for _, op := range flushed[0].ops { + assert.Contains(t, op.srcKey, "s10-", "revision 11 ops must not leak into revision 10's flush") + } + + rest := b.flush() + require.NotNil(t, rest) + assert.Len(t, rest.ops, 3) + assert.EqualValues(t, 11, rest.watermark) +} + +// TestBatcherByteWatermark: the TxnFlushBytes boundary triggers a flush even +// when the op count is far below MaxTxnOps — the pending set flushes alone +// (at its revision boundary) when the next revision would push it past the +// byte watermark. +func TestBatcherByteWatermark(t *testing.T) { + b := newBatcher(100, 1000) + require.Empty(t, b.add(mkGroup(1, 1, 400))) + + flushed := b.add(mkGroup(2, 1, 700)) + require.Len(t, flushed, 1, "revision 2 does not fit on top: revision 1 must flush first") + assert.Len(t, flushed[0].ops, 1) + assert.EqualValues(t, 1, flushed[0].watermark) + assert.False(t, flushed[0].oversized) + + rest := b.flush() + require.NotNil(t, rest) + assert.Len(t, rest.ops, 1) + assert.EqualValues(t, 2, rest.watermark) +} + +// TestBatcherOversizedSingleRevision: one revision alone above both +// watermarks becomes its own flush set, marked oversized, never split. +func TestBatcherOversizedSingleRevision(t *testing.T) { + b := newBatcher(4, 100) // 3 data op slots + flushed := b.add(mkGroup(7, 9, 50)) + require.Len(t, flushed, 1, "an oversized revision must flush immediately as one set") + fs := flushed[0] + assert.True(t, fs.oversized) + assert.Len(t, fs.ops, 9, "all 9 ops of the revision stay in ONE Txn") + assert.EqualValues(t, 7, fs.watermark) + assert.Equal(t, "s7-8", fs.lastSrcKey) + assert.Nil(t, b.flush()) +} + +// TestBatcherOversizedDoesNotDragNeighbors: pending small revisions flush +// separately before an oversized revision arrives. +func TestBatcherOversizedDoesNotDragNeighbors(t *testing.T) { + b := newBatcher(10, 1<<20) + require.Empty(t, b.add(mkGroup(1, 2, 1))) + + flushed := b.add(mkGroup(2, 20, 1)) + require.Len(t, flushed, 2, "pending set flushes first, then the oversized revision alone") + assert.Len(t, flushed[0].ops, 2) + assert.False(t, flushed[0].oversized) + assert.Len(t, flushed[1].ops, 20) + assert.True(t, flushed[1].oversized) +} + +// TestBatcherFlushesOnDuplicateKey: etcd rejects duplicate keys within one +// Txn, and an unsynced-watcher catch-up response batches up to 1000 +// revisions — so a key modified twice in the window must split the flush at +// the revision boundary instead of producing a poison Txn. +func TestBatcherFlushesOnDuplicateKey(t *testing.T) { + b := newBatcher(100, 1<<20) + g1 := revGroup{rev: 5, ops: []kvOp{{key: "/dst/a", value: "v1"}, {key: "/dst/b", value: "v1"}}} + require.Empty(t, b.add(g1)) + + // Revision 6 touches /dst/a again: revision 5 must flush alone first. + g2 := revGroup{rev: 6, ops: []kvOp{{key: "/dst/a", value: "v2"}}} + flushed := b.add(g2) + require.Len(t, flushed, 1, "a duplicate key must force a flush at the revision boundary") + assert.EqualValues(t, 5, flushed[0].watermark) + assert.Len(t, flushed[0].ops, 2) + + rest := b.flush() + require.NotNil(t, rest) + assert.Len(t, rest.ops, 1) + assert.EqualValues(t, 6, rest.watermark) + + // Disjoint keys still coalesce. + require.Empty(t, b.add(revGroup{rev: 7, ops: []kvOp{{key: "/dst/c"}}})) + require.Empty(t, b.add(revGroup{rev: 8, ops: []kvOp{{key: "/dst/d"}}})) + both := b.flush() + require.NotNil(t, both) + assert.Len(t, both.ops, 2, "distinct keys must keep coalescing across revisions") +} + +// TestBatcherFlushSetRetainsGroups: revision boundaries survive into the +// flush set, so a target-limit rejection can be re-committed at revision +// granularity. +func TestBatcherFlushSetRetainsGroups(t *testing.T) { + b := newBatcher(100, 1<<20) + require.Empty(t, b.add(mkGroup(1, 2, 1))) + require.Empty(t, b.add(mkGroup(2, 3, 1))) + fs := b.flush() + require.NotNil(t, fs) + require.Len(t, fs.groups, 2) + assert.EqualValues(t, 1, fs.groups[0].rev) + assert.Len(t, fs.groups[0].ops, 2) + assert.EqualValues(t, 2, fs.groups[1].rev) + assert.Len(t, fs.groups[1].ops, 3) +} + +func TestGroupByRevision(t *testing.T) { + ops := []kvOp{ + {key: "a"}, {key: "b"}, // rev 5 + {key: "c"}, // rev 6 + {key: "d"}, {key: "e"}, // rev 9 + } + revs := []int64{5, 5, 6, 9, 9} + groups := groupByRevision(ops, revs) + require.Len(t, groups, 3) + assert.EqualValues(t, 5, groups[0].rev) + assert.Len(t, groups[0].ops, 2) + assert.EqualValues(t, 6, groups[1].rev) + assert.Len(t, groups[1].ops, 1) + assert.EqualValues(t, 9, groups[2].rev) + assert.Len(t, groups[2].ops, 2) + + assert.Nil(t, groupByRevision(nil, nil)) + assert.Nil(t, groupByRevision(ops, revs[:2]), "length mismatch must yield nothing") +} diff --git a/pkg/mirroragent/client.go b/pkg/mirroragent/client.go new file mode 100644 index 00000000..6974c7de --- /dev/null +++ b/pkg/mirroragent/client.go @@ -0,0 +1,62 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "crypto/tls" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +// Client is the subset of *clientv3.Client the engine uses on each side. +// The source side needs KV (Get) + Watcher + Status; the target side needs +// KV (Get/Txn) + Status. *clientv3.Client satisfies it directly. +type Client interface { + clientv3.KV + clientv3.Watcher + Status(ctx context.Context, endpoint string) (*clientv3.StatusResponse, error) + Endpoints() []string +} + +var _ Client = (*clientv3.Client)(nil) + +// Keepalive settings the engine's liveness machinery assumes: without +// client-driven keepalives, NLB (350s) and Cloud NAT (1200s) idle timeouts +// silently kill quiet watches on the cross-cloud path this engine exists +// for. +const ( + DialKeepAliveTime = 25 * time.Second + DialKeepAliveTimeout = 10 * time.Second +) + +// NewClientConfig returns a clientv3.Config wired the way the engine +// requires: keepalives on (including without active streams) and a bounded +// dial. Callers own TLS/auth material — this library never reads Secrets. +// Per-unary request deadlines are applied inside the engine from +// Config.RequestTimeout, not here. +func NewClientConfig(endpoints []string, tlsConfig *tls.Config, dialTimeout time.Duration) clientv3.Config { + return clientv3.Config{ + Endpoints: endpoints, + DialTimeout: dialTimeout, + DialKeepAliveTime: DialKeepAliveTime, + DialKeepAliveTimeout: DialKeepAliveTimeout, + PermitWithoutStream: true, + TLS: tlsConfig, + } +} diff --git a/pkg/mirroragent/config.go b/pkg/mirroragent/config.go new file mode 100644 index 00000000..26a58105 --- /dev/null +++ b/pkg/mirroragent/config.go @@ -0,0 +1,346 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// Mode selects the agent's operating mode. Mirrors EtcdMirrorMode in +// api/v1alpha1. +type Mode string + +const ( + // ModeSync is normal continuous replication. + ModeSync Mode = "Sync" + // ModeDrain prepares for cutover: the agent records the source revision + // observed when the drain starts, keeps replicating until the checkpoint + // watermark reaches it, runs a verification pass, then flips the fence + // key's role to Primary so any straggler apply fails its mod-revision + // compare loudly, and returns from Run. + ModeDrain Mode = "Drain" +) + +// InitialSyncMode governs pre-existing keys under the effective destination +// prefix at genesis. Mirrors EtcdMirrorInitialSyncMode in api/v1alpha1. +type InitialSyncMode string + +const ( + // InitialSyncRequireEmpty refuses to start if the destination prefix + // already holds any key (the reserved checkpoint key is excluded by + // exact match). Re-arms whenever the checkpoint is invalidated by a + // cluster-identity mismatch. + InitialSyncRequireEmpty InitialSyncMode = "RequireEmpty" + // InitialSyncOverwrite scans and writes over whatever is there. Keys + // present on the target but absent on the source are left alone. + InitialSyncOverwrite InitialSyncMode = "Overwrite" + // InitialSyncOverwriteAndPrune is Overwrite plus one mandatory + // orphan-prune pass after the scan, making reversal onto a + // previously-populated prefix (failback) a first-class correct operation. + InitialSyncOverwriteAndPrune InitialSyncMode = "OverwriteAndPrune" +) + +// DefaultCheckpointKeySuffix is appended to the effective destination prefix +// to form the default reserved checkpoint key. The \x00 byte after the +// prefix cannot collide with any real key under it. +const DefaultCheckpointKeySuffix = "\x00etcdmirror-checkpoint" + +// Defaults, matching the CRD field defaults in api/v1alpha1. +const ( + DefaultMaxTxnOps = 128 + DefaultTxnFlushBytes = 1 << 20 // 1Mi + DefaultPageKeyLimit = 512 + DefaultPageBytes = 1 << 20 // 1Mi + DefaultRequestTimeout = 30 * time.Second + DefaultBackoffInitial = 1 * time.Second + DefaultBackoffMax = 30 * time.Second + // DefaultReconcilePeriod is the spec→Config translation default that + // cmd/mirror-agent applies for spec.reconciliation.enabled with a nil + // interval; the engine itself treats 0 as disabled. + DefaultReconcilePeriod = time.Hour + + // DefaultWatchBufferBytes bounds the in-memory replay buffer for watch + // events observed from R0 while the genesis scan runs. Must stay in + // lockstep with the EtcdMirror CRD's spec.sync.watchBufferBytes default. + DefaultWatchBufferBytes = 16 << 20 // 16Mi + // DefaultProgressInterval is how often the agent issues a client-driven + // RequestProgress on the source watch (server-side notify intervals are + // uncontrollable on foreign clusters). + DefaultProgressInterval = 45 * time.Second + // DefaultResyncLoopThreshold is how many consecutive forced resyncs + // (without reaching steady state in between) trip the livelock detector + // — the signature of source retention < scan+drain time. + DefaultResyncLoopThreshold = 3 + // DefaultQuotaProbeInterval is how often a quota-exhausted (NOSPACE) + // agent re-probes the target. Deliberately a slow flat poll, not + // backoff: quota exhaustion only heals when an operator acts. + DefaultQuotaProbeInterval = time.Minute +) + +// Config is the engine's plain-Go configuration. Fields mirror the +// EtcdMirror CRD spec (api/v1alpha1/etcdmirror_types.go); doc comments here +// and there are the PR1<->PR2 alignment contract. +type Config struct { + // LinkUID uniquely identifies this mirror link (source, target, prefix + // tuple); typically the EtcdMirror object's UID. Stamped into the fence + // key: a checkpoint carrying a different LinkUID is another link's fence + // and the agent refuses to touch it. Required. + LinkUID string + + // Epoch is this agent generation within the link, monotonically + // increased by the supervisor on each re-deploy. An agent that finds a + // higher epoch in the fence stops permanently (a newer generation owns + // the link); a lower stored epoch is taken over via the fenced write + // path. Must be >= 1. + Epoch int64 + + // Mode selects continuous replication (Sync, the default) or a cutover + // drain (Drain). See ModeDrain; RequestDrain flips a running agent. + Mode Mode + + // SourcePrefix scopes which source keys are mirrored; empty means the + // whole keyspace. + SourcePrefix string + // TargetPrefix is the prefix under which mirrored keys land. + TargetPrefix string + // DestPrefix is the middle term of the rewrite formula + // + // key' = TargetPrefix + DestPrefix + TrimPrefix(key, SourcePrefix) + // + // Default "" means the source prefix is stripped and key remainders land + // directly under TargetPrefix. + DestPrefix string + // ExcludePrefixes lists source key prefixes (full source-side keys) + // skipped entirely: not scanned, not watched, not counted, not pruned. + // Nested or duplicate entries are normalized away at defaulting time (a + // prefix covered by another is dropped) so range subtraction and count + // corrections each see a disjoint set. + ExcludePrefixes []string + + // InitialSyncMode governs pre-existing destination keys at genesis. + // Defaults to RequireEmpty. + InitialSyncMode InitialSyncMode + // StartRevision, when > 0, skips the genesis scan entirely and starts + // watching from StartRevision+1 (for fidelity-preserving snapshot + // seeds). Requires InitialSyncMode Overwrite or OverwriteAndPrune. + StartRevision int64 + + // CheckpointKey overrides the reserved checkpoint/fence key on the + // target. Defaults to the effective destination prefix + + // DefaultCheckpointKeySuffix. The key is excluded by exact match from + // scans, counts, prune passes, and the RequireEmpty check; the target + // RBAC grant must cover it. + CheckpointKey string + + // MaxTxnOps bounds how many operations the agent batches into a single + // target Txn, including the reserved checkpoint-write slot. Must not + // exceed the target's --max-txn-ops. Defaults to 128; minimum 2. + MaxTxnOps int + // TxnFlushBytes is the byte watermark at which a batch is flushed (at + // the next source-revision boundary). Defaults to 1Mi. + TxnFlushBytes int64 + // PageKeyLimit bounds keys per source scan page. The scan is pull-based, + // one page in flight — no read-ahead. Defaults to 512. + PageKeyLimit int + // PageBytes bounds bytes per source scan page. etcd Range has no byte + // limit, so this is enforced adaptively: the next page's key limit is + // derived from the observed bytes/key of the previous page. Defaults + // to 1Mi. + PageBytes int64 + // MaxOpsPerSecond rate-limits target writes (puts+deletes/sec, token + // bucket), applied to both the genesis scan and watch-driven applies. + // Zero means unlimited. + MaxOpsPerSecond int + // RequestTimeout is the per-RPC context deadline applied to every unary + // call on both sides (watches excluded — they are long-lived by design + // and covered by progress-notification liveness instead). Defaults + // to 30s. + RequestTimeout time.Duration + + // BackoffInitialDelay/BackoffMaxDelay bound the retry loop for + // connection-class errors. Throttling-class errors use a more + // conservative curve derived from the same bounds; quota exhaustion and + // permanent errors are never retried through this loop. Defaults: + // 1s to 30s. + BackoffInitialDelay time.Duration + BackoffMaxDelay time.Duration + + // ReconcileInterval > 0 enables the periodic full diff-and-repair pass, + // executed inline on the steady-state tail loop — never concurrently + // with the genesis scan, a forced-resync sweep, or a drain (a requested + // drain's own verification supersedes it) — and re-scheduled a full + // interval after each periodic pass and after the genesis/forced-resync + // sweep (a drain's verification is terminal and never re-arms; a + // requested drain gates the periodic pass anyway), keeping the key + // counts within the CRD's 2x-interval freshness contract whenever + // passes complete faster than the interval. Independent of this, one + // reconciliation-with-delete pass always runs after any forced resync + // (mark-and-sweep), as the OverwriteAndPrune genesis pass, and before + // the Drain verification. 0 disables the periodic pass (the CRD's + // spec.reconciliation.enabled maps to this; DefaultReconcilePeriod is + // the translation default for Enabled with a nil interval). + ReconcileInterval time.Duration + // ReconcileDeleteOrphans allows the PERIODIC pass to delete target keys + // with no source counterpart; when false the pass still repairs missing + // and divergent keys and reports orphans in the drift. Forced-resync + // sweeps and OverwriteAndPrune always delete orphans. + ReconcileDeleteOrphans bool + + // WatchBufferBytes bounds the memory used to buffer watch events + // observed from R0 while the genesis scan runs (the reflector replay + // buffer). On overflow the agent cancels the source watch and restarts + // the scan from a fresh R0 — a bounded retry instead of unbounded growth + // when source churn outruns scan+apply throughput. The restart is + // surfaced as Snapshot.LastScanRestartCause WatchBufferOverflow (the + // controller maps it to the InitialSyncCompactionRaced event) and + // repeated overflows count toward the resync-loop detector. Defaults to + // DefaultWatchBufferBytes; must stay in lockstep with the CRD's + // spec.sync.watchBufferBytes. Must be >= 0 (0 = default). + WatchBufferBytes int64 + + // Agent-internal knobs (not part of the CRD in v1). + ProgressInterval time.Duration + ResyncLoopThreshold int + QuotaProbeInterval time.Duration +} + +// withDefaults returns a copy with zero fields replaced by defaults. +func (c Config) withDefaults() Config { + if c.Mode == "" { + c.Mode = ModeSync + } + if c.InitialSyncMode == "" { + c.InitialSyncMode = InitialSyncRequireEmpty + } + if c.CheckpointKey == "" { + c.CheckpointKey = c.EffectiveDestPrefix() + DefaultCheckpointKeySuffix + } + if c.MaxTxnOps == 0 { + c.MaxTxnOps = DefaultMaxTxnOps + } + if c.TxnFlushBytes == 0 { + c.TxnFlushBytes = DefaultTxnFlushBytes + } + if c.PageKeyLimit == 0 { + c.PageKeyLimit = DefaultPageKeyLimit + } + if c.PageBytes == 0 { + c.PageBytes = DefaultPageBytes + } + if c.RequestTimeout == 0 { + c.RequestTimeout = DefaultRequestTimeout + } + if c.BackoffInitialDelay == 0 { + c.BackoffInitialDelay = DefaultBackoffInitial + } + if c.BackoffMaxDelay == 0 { + c.BackoffMaxDelay = DefaultBackoffMax + } + if c.WatchBufferBytes == 0 { + c.WatchBufferBytes = DefaultWatchBufferBytes + } + if c.ProgressInterval == 0 { + c.ProgressInterval = DefaultProgressInterval + } + if c.ResyncLoopThreshold == 0 { + c.ResyncLoopThreshold = DefaultResyncLoopThreshold + } + if c.QuotaProbeInterval == 0 { + c.QuotaProbeInterval = DefaultQuotaProbeInterval + } + c.ExcludePrefixes = normalizePrefixes(c.ExcludePrefixes) + return c +} + +// normalizePrefixes sorts prefixes and drops any entry covered by another +// (nested or duplicate). Both the scan-range subtraction and the per-prefix +// count corrections assume a disjoint set: a key covered by two overlapping +// entries must never be subtracted from a count twice. +func normalizePrefixes(in []string) []string { + if len(in) < 2 { + return in + } + sorted := make([]string, len(in)) + copy(sorted, in) + sort.Strings(sorted) + out := sorted[:0] + for _, p := range sorted { + if len(out) > 0 && strings.HasPrefix(p, out[len(out)-1]) { + continue + } + out = append(out, p) + } + return out +} + +// Validate checks the configuration after defaulting. +func (c Config) Validate() error { + if c.LinkUID == "" { + return fmt.Errorf("linkUID is required") + } + if c.Epoch < 1 { + return fmt.Errorf("epoch must be >= 1, got %d", c.Epoch) + } + if c.Mode != ModeSync && c.Mode != ModeDrain { + return fmt.Errorf("invalid mode %q", c.Mode) + } + switch c.InitialSyncMode { + case InitialSyncRequireEmpty, InitialSyncOverwrite, InitialSyncOverwriteAndPrune: + default: + return fmt.Errorf("invalid initialSyncMode %q", c.InitialSyncMode) + } + if c.StartRevision < 0 { + return fmt.Errorf("startRevision must be >= 0, got %d", c.StartRevision) + } + if c.StartRevision > 0 && c.InitialSyncMode == InitialSyncRequireEmpty { + return fmt.Errorf("startRevision requires initialSyncMode Overwrite or OverwriteAndPrune") + } + if c.MaxTxnOps < 2 { + return fmt.Errorf("maxTxnOps must be >= 2 (one op slot is reserved for the checkpoint), got %d", c.MaxTxnOps) + } + if c.TxnFlushBytes < 1 || c.PageBytes < 1 || c.PageKeyLimit < 1 { + return fmt.Errorf("txnFlushBytes, pageBytes and pageKeyLimit must be positive") + } + if c.MaxOpsPerSecond < 0 { + return fmt.Errorf("maxOpsPerSecond must be >= 0, got %d", c.MaxOpsPerSecond) + } + if c.ReconcileInterval < 0 { + return fmt.Errorf("reconcileInterval must be >= 0, got %v", c.ReconcileInterval) + } + if c.WatchBufferBytes < 0 { + return fmt.Errorf("watchBufferBytes must be >= 0, got %d", c.WatchBufferBytes) + } + if !strings.HasPrefix(c.CheckpointKey, c.EffectiveDestPrefix()) { + return fmt.Errorf("checkpointKey must live under the effective destination prefix") + } + for _, p := range c.ExcludePrefixes { + if p == "" { + return fmt.Errorf("excludePrefixes entries must be non-empty") + } + } + return nil +} + +// EffectiveDestPrefix is TargetPrefix + DestPrefix: the target-side prefix +// every mirrored key lands under, and the range RequireEmpty and prune +// passes operate on. +func (c Config) EffectiveDestPrefix() string { + return c.TargetPrefix + c.DestPrefix +} diff --git a/pkg/mirroragent/config_test.go b/pkg/mirroragent/config_test.go new file mode 100644 index 00000000..85e30140 --- /dev/null +++ b/pkg/mirroragent/config_test.go @@ -0,0 +1,157 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func minimalCfg() Config { + return Config{LinkUID: "l", Epoch: 1, SourcePrefix: "/s/", TargetPrefix: "/d/"} +} + +func TestConfigDefaults(t *testing.T) { + c := minimalCfg().withDefaults() + require.NoError(t, c.Validate()) + + assert.Equal(t, ModeSync, c.Mode) + assert.Equal(t, InitialSyncRequireEmpty, c.InitialSyncMode) + assert.Equal(t, "/d/"+DefaultCheckpointKeySuffix, c.CheckpointKey) + assert.Equal(t, DefaultMaxTxnOps, c.MaxTxnOps) + assert.EqualValues(t, DefaultTxnFlushBytes, c.TxnFlushBytes) + assert.Equal(t, DefaultPageKeyLimit, c.PageKeyLimit) + assert.EqualValues(t, DefaultPageBytes, c.PageBytes) + assert.Equal(t, DefaultRequestTimeout, c.RequestTimeout) + assert.EqualValues(t, DefaultWatchBufferBytes, c.WatchBufferBytes) + assert.Equal(t, DefaultResyncLoopThreshold, c.ResyncLoopThreshold) + assert.Equal(t, DefaultProgressInterval, c.ProgressInterval) + assert.Equal(t, DefaultQuotaProbeInterval, c.QuotaProbeInterval) +} + +func TestConfigValidate(t *testing.T) { + cases := []struct { + name string + mutate func(*Config) + wantErr string + }{ + {name: "valid", mutate: func(*Config) {}}, + {name: "missing linkUID", mutate: func(c *Config) { c.LinkUID = "" }, + wantErr: "linkUID"}, + {name: "epoch below one", mutate: func(c *Config) { c.Epoch = 0 }, + wantErr: "epoch"}, + {name: "bad mode", mutate: func(c *Config) { c.Mode = "Paused" }, + wantErr: "mode"}, + {name: "bad initialSyncMode", mutate: func(c *Config) { c.InitialSyncMode = "Merge" }, + wantErr: "initialSyncMode"}, + {name: "negative startRevision", mutate: func(c *Config) { c.StartRevision = -1 }, + wantErr: "startRevision"}, + // Defense-in-depth mirror of the CRD's CEL rule: a startRevision + // seed skips the scan, so RequireEmpty could never be satisfied + // meaningfully. + {name: "startRevision requires overwrite", mutate: func(c *Config) { c.StartRevision = 10 }, + wantErr: "startRevision requires initialSyncMode Overwrite"}, + {name: "startRevision with overwrite ok", mutate: func(c *Config) { + c.StartRevision = 10 + c.InitialSyncMode = InitialSyncOverwrite + }}, + {name: "maxTxnOps below two", mutate: func(c *Config) { c.MaxTxnOps = 1 }, + wantErr: "maxTxnOps must be >= 2"}, + {name: "negative txnFlushBytes", mutate: func(c *Config) { c.TxnFlushBytes = -1 }, + wantErr: "positive"}, + {name: "negative pageBytes", mutate: func(c *Config) { c.PageBytes = -1 }, + wantErr: "positive"}, + {name: "negative pageKeyLimit", mutate: func(c *Config) { c.PageKeyLimit = -1 }, + wantErr: "positive"}, + {name: "negative maxOpsPerSecond", mutate: func(c *Config) { c.MaxOpsPerSecond = -1 }, + wantErr: "maxOpsPerSecond"}, + {name: "negative watchBufferBytes", mutate: func(c *Config) { c.WatchBufferBytes = -1 }, + wantErr: "watchBufferBytes"}, + {name: "checkpoint key outside dest prefix", mutate: func(c *Config) { c.CheckpointKey = "/elsewhere" }, + wantErr: "checkpointKey"}, + {name: "empty exclude entry", mutate: func(c *Config) { c.ExcludePrefixes = []string{""} }, + wantErr: "excludePrefixes"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := minimalCfg() + tc.mutate(&c) + err := c.withDefaults().Validate() + if tc.wantErr == "" { + assert.NoError(t, err) + return + } + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// TestConfigValidateReconcileInterval: negative intervals are rejected, 0 +// (disabled) and positive intervals pass, and ReconcileDeleteOrphans with a +// zero interval is accepted as the documented no-op. +func TestConfigValidateReconcileInterval(t *testing.T) { + c := minimalCfg() + c.ReconcileInterval = -time.Second + err := c.withDefaults().Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "reconcileInterval") + + c = minimalCfg() + assert.NoError(t, c.withDefaults().Validate(), "zero interval (disabled) must validate") + + c = minimalCfg() + c.ReconcileInterval = time.Minute + assert.NoError(t, c.withDefaults().Validate()) + + c = minimalCfg() + c.ReconcileDeleteOrphans = true + assert.NoError(t, c.withDefaults().Validate(), + "deleteOrphans with the periodic pass disabled is a documented no-op, not an error") +} + +// TestNormalizePrefixes: nested/duplicate exclude entries are collapsed at +// defaulting time — count corrections assume a disjoint set, and a key +// covered by two overlapping entries must never be subtracted twice +// (previously a byte-exact converged drain could fail verification). +func TestNormalizePrefixes(t *testing.T) { + c := minimalCfg() + c.ExcludePrefixes = []string{"/s/tmp/cache/", "/s/tmp/", "/s/other/", "/s/tmp/"} + c = c.withDefaults() + require.NoError(t, c.Validate()) + assert.Equal(t, []string{"/s/other/", "/s/tmp/"}, c.ExcludePrefixes) + + c2 := minimalCfg() + c2.ExcludePrefixes = []string{"/s/a/"} + assert.Equal(t, []string{"/s/a/"}, c2.withDefaults().ExcludePrefixes) + c3 := minimalCfg() + assert.Empty(t, c3.withDefaults().ExcludePrefixes) +} + +// TestCheckpointKeyConvention: the default reserved key uses the +// \x00-after-prefix convention, which no real key under the prefix can +// collide with, and lives under the effective destination prefix. +func TestCheckpointKeyConvention(t *testing.T) { + c := Config{LinkUID: "l", Epoch: 1, SourcePrefix: "/s/", TargetPrefix: "/d/", DestPrefix: "sub/"} + c = c.withDefaults() + require.NoError(t, c.Validate()) + assert.Equal(t, "/d/sub/", c.EffectiveDestPrefix()) + assert.Equal(t, "/d/sub/\x00etcdmirror-checkpoint", c.CheckpointKey) +} diff --git a/pkg/mirroragent/doc.go b/pkg/mirroragent/doc.go new file mode 100644 index 00000000..95fa3427 --- /dev/null +++ b/pkg/mirroragent/doc.go @@ -0,0 +1,101 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package mirroragent implements the EtcdMirror replication engine: a +// continuous, one-way key-range sync from a source etcd cluster into a +// target etcd cluster. It is a pure library — no Kubernetes API types, no +// binary, no metrics endpoint; progress is exposed through [Agent.Snapshot]. +// +// Engine invariants (the doc comments in api/v1alpha1/etcdmirror_types.go +// state the same contracts on the CRD side; keep both in sync): +// +// - Genesis is an UNPINNED chunked scan with the watch already open from +// the revision observed before the scan started; buffered events are +// replayed over the scanned base (reflector pattern). Pages read at the +// current revision, so mid-scan compaction cannot fail the scan. +// - The checkpoint (source-revision watermark plus {linkUID, epoch, role}) +// lives IN THE TARGET etcd at a reserved key, written in the SAME Txn as +// every applied batch and fenced with a mod-revision compare on EVERY +// write path (applies, reconciliation repairs, prune deletes). The +// reserved key is excluded by exact match from scans, counts, prune +// passes, and the RequireEmpty check. +// - Target Txns flush ONLY at source-revision boundaries: a source +// revision's events are never split across Txns, whole revisions are +// coalesced up to the MaxTxnOps/TxnFlushBytes watermarks, and one op +// slot is always reserved for the checkpoint write. A single revision +// larger than MaxTxnOps is applied as one oversized Txn with the +// checkpoint held until it lands. +// - Key rewrite is one formula, anchored, never a substring replace: +// key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix). +// - Errors are classified (see [Classify]): compaction forces a resync +// (with a livelock detector), NOSPACE parks the agent until an operator +// acts, oversized requests are permanent with the offending key surfaced +// redacted (see [RedactKey]; values never logged), throttling backs off +// on its own curve. +// - Memory is bounded: one in-flight scan page (byte-bounded, adaptive) +// and a byte-bounded replay buffer for the watch opened before the scan; +// on overflow the agent cancels the source watch and restarts the scan +// from a fresh R0 (a bounded retry, surfaced with cause +// WatchBufferOverflow) instead of growing. +// +// # Why mid-scan compaction cannot wedge the engine +// +// 1. At scan start one linearizable Get (WithCountOnly) returns +// Header.Revision = R0 and the total Count in a single RPC. +// 2. The watch opens at R0+1 BEFORE any scan page is read. R0 was observed +// this instant by a linearizable read, so it cannot already be compacted. +// 3. Every scan page is an UNPINNED Get (no WithRev). ErrCompacted is a +// property of reads pinned below the compact revision; an unpinned read +// is immune by construction, at every point during the scan, regardless +// of concurrent compactions. The failure class is removed, not detected +// and retried. +// 4. The only remaining compaction hazard is the watch stream itself going +// quiet long enough that a re-Watch(WithRev(watermark+1)) lands below +// the compact revision — identical in shape during InitialSync and +// steady state, handled by one mechanism: forced resync with a +// mandatory mark-and-sweep prune. +// +// Scan and watch are not sequential phases with a handoff race: the watch is +// live for the entire scan, and scan writes may interleave with replayed +// watch writes because both write the same idempotent final value for a +// given key — convergence to the last-write value; a duplicate Put of +// identical content is a correctness no-op, only a bounded efficiency cost. +// +// # Why the fence needs only a mod_revision compare +// +// etcd's Compare supports whole-value/mod_revision/version/create_revision +// predicates only — no field-level JSON predicates. All safety rests on one +// discipline, identical on EVERY write path (apply, reconcile repair, prune, +// cutover role-flip): +// +// If(Compare(ModRevision(fenceKey), "=", observedModRev)). +// Then(dataOps..., Put(fenceKey, next)) +// // on !Succeeded: re-read, recompute, retry — never blind re-Commit +// +// linkUID/epoch/role are payload for humans and the engine's state machine, +// never comparison predicates. Cutover safety falls out for free: the +// role-flip Txn bumps ModRevision, so any writer holding a pre-flip +// observedModRev fails its next compare loudly — indistinguishable from an +// ordinary concurrent-writer collision, no special role-check code needed. +// +// # Retry ownership +// +// clientv3 auto-retries Get only on codes.Unavailable; Txn/Put/Delete are +// write-at-most-once (client-retried only when no connection was ever +// established). The engine owns 100% of write-path retry/backoff. A +// fenced-Txn retry after an ambiguous timeout must re-read the fence first — +// the Txn's own success bumped the fence ModRevision. +package mirroragent diff --git a/pkg/mirroragent/errors.go b/pkg/mirroragent/errors.go new file mode 100644 index 00000000..3e1343c0 --- /dev/null +++ b/pkg/mirroragent/errors.go @@ -0,0 +1,329 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" +) + +// Class is the engine's error taxonomy. Every failure the retry loop sees is +// classified into exactly one class, and each class has its own recovery +// policy — misclassification (e.g. labelling an oversized Txn "throttling") +// is itself a bug class this taxonomy exists to eliminate. +type Class string + +const ( + // ClassTransient covers connection-class errors (unavailable, timeouts, + // leader loss): retried with the standard exponential backoff. + ClassTransient Class = "Transient" + // ClassThrottle covers the target rejecting the write rate + // (rpctypes.ErrTooManyRequests and other rate-flavored + // ResourceExhausted): retried on a more conservative curve, distinct + // from ClassTransient and never conflated with ClassQuota. + ClassThrottle Class = "Throttle" + // ClassResync covers conditions that invalidate the watch/checkpoint + // position (source compaction outran the watch, a bound cluster identity + // changed): the agent runs a forced resync (scan + mandatory + // mark-and-sweep), counted by the resync-loop livelock detector. + ClassResync Class = "Resync" + // ClassQuota is rpctypes.ErrNoSpace from the target: permanent until an + // operator compacts/defrags/disarms. The agent parks on a slow flat + // probe instead of burning backoff against a full quota. + ClassQuota Class = "Quota" + // ClassPermanent is never retried: oversized requests (server + // "request is too large"/"too many operations" and the client 2MiB + // send cap), auth/permission misconfiguration, fence violations, + // corrupt/unknown-version checkpoints, version-floor failures, + // RequireEmpty violations. + ClassPermanent Class = "Permanent" +) + +// ResyncReason distinguishes why a forced resync was required. +type ResyncReason string + +const ( + // ResyncReasonCompacted: source compaction outran the watch (restart or + // pause longer than retention). + ResyncReasonCompacted ResyncReason = "Compacted" + // ResyncReasonClusterIDMismatch: a bound cluster ID (source or target) + // no longer matches the probed cluster; genesis is forced and + // RequireEmpty re-arms. + ResyncReasonClusterIDMismatch ResyncReason = "ClusterIDMismatch" +) + +// ResyncError forces a full resync (genesis scan + mark-and-sweep). +type ResyncError struct { + Reason ResyncReason + Cause error +} + +func (e *ResyncError) Error() string { + return fmt.Sprintf("forced resync required (%s): %v", e.Reason, e.Cause) +} +func (e *ResyncError) Unwrap() error { return e.Cause } + +// CheckpointInvalidError reports a corrupt or unknown-version checkpoint. +// Distinct from an absent checkpoint (plain genesis). PERMANENT: an +// undecodable fence cannot prove link ownership, epoch ordering, or role, so +// overwriting it (and running a resync's mandatory prune over data it may be +// protecting) is never safe. The operator must inspect the reserved key and +// delete it to recover. +type CheckpointInvalidError struct { + Reason string +} + +func (e *CheckpointInvalidError) Error() string { + return "checkpoint invalid: " + e.Reason +} + +// ConfigError reports a client/engine configuration defect detected at +// runtime (e.g. a client with no endpoints). Permanent. +type ConfigError struct { + Detail string +} + +func (e *ConfigError) Error() string { return "configuration error: " + e.Detail } + +// FenceError is a fence violation: the reserved key's mod revision moved +// under us (another agent generation took over, or the role flipped to +// Primary at cutover). Permanent — this agent must never write again. +type FenceError struct { + Detail string +} + +func (e *FenceError) Error() string { return "fence violation: " + e.Detail } + +// RedactKey returns a safe display form for a key surfaced in status, +// events, or logs: the configured destination prefix (already public in +// the spec) + "…" + the first 8 hex chars of sha256(key). Key bytes beyond +// the prefix are never surfaced; values never at all. +func RedactKey(prefix string, key []byte) string { + sum := sha256.Sum256(key) + return prefix + "…" + hex.EncodeToString(sum[:])[:8] +} + +// TooLargeError is an oversized request: the server's Txn size or op-count +// limit, or the gRPC client send cap. Permanent; carries the offending key +// (never the value) so operators can find the poison key. +type TooLargeError struct { + // Key is the redacted form (RedactKey) of the first key of the offending + // batch (target-side): the destination prefix plus a hash — raw key + // suffixes and values are deliberately never carried. + Key string + Ops int + Bytes int64 + Cause error +} + +func (e *TooLargeError) Error() string { + return fmt.Sprintf("request too large (%d ops, %d bytes, first key %q): %v", + e.Ops, e.Bytes, e.Key, e.Cause) +} +func (e *TooLargeError) Unwrap() error { return e.Cause } + +// EmptyTargetViolationError reports a non-empty destination prefix under +// InitialSyncRequireEmpty. Permanent. The range identifies exactly what an +// operator must clear (`etcdctl del` over [RangeStart, RangeEnd)); the +// reserved checkpoint key was excluded from the count. +type EmptyTargetViolationError struct { + RangeStart string + RangeEnd string + KeyCount int64 +} + +func (e *EmptyTargetViolationError) Error() string { + return fmt.Sprintf( + "destination prefix not empty: %d pre-existing keys in [%q, %q) and initialSyncMode is RequireEmpty", + e.KeyCount, e.RangeStart, e.RangeEnd) +} + +// PrefixConflictError reports another EtcdMirror link's reserved fence key +// found inside this link's effective destination prefix during a prune pass: +// two links target overlapping destination ranges on the same cluster. +// Deleting the sibling's fence (and its data, as "orphans") would silently +// destroy the other link, so the pass stops loudly instead. Permanent until +// the operator resolves the overlap. +type PrefixConflictError struct { + // Key is the redacted form (RedactKey) of the foreign reserved key. + Key string + // OwnerLinkUID is the link that owns the foreign fence. + OwnerLinkUID string +} + +func (e *PrefixConflictError) Error() string { + return fmt.Sprintf( + "destination prefix conflict: reserved fence key %q under this link's destination prefix belongs to link %q", + e.Key, e.OwnerLinkUID) +} + +// DrainVerificationError reports a post-drain per-side key-count mismatch +// that one repair pass did not resolve. Permanent — cutover must not proceed +// on divergent data. +type DrainVerificationError struct { + SourceKeys int64 + TargetKeys int64 +} + +func (e *DrainVerificationError) Error() string { + return fmt.Sprintf("drain verification failed: source has %d keys, target has %d", + e.SourceKeys, e.TargetKeys) +} + +// UnsupportedVersionError reports an etcd server below the declared >=3.4 +// hard floor. Permanent. +type UnsupportedVersionError struct { + Side string // "source" or "target" + Version string +} + +func (e *UnsupportedVersionError) Error() string { + return fmt.Sprintf("%s etcd version %s is below the supported floor %s", e.Side, e.Version, hardVersionFloor) +} + +// ReasonFor maps a typed engine error to its API-aligned condition/phase +// reason string ("" for untyped errors). The controller reads this off +// Snapshot.LastErrorReason instead of matching LastError message substrings, +// which would break on any message edit. Same errors.As ladder as Classify. +func ReasonFor(err error) string { + if err == nil { + return "" + } + var ( + cpInvalid *CheckpointInvalidError + fenceErr *FenceError + tooLarge *TooLargeError + emptyTarget *EmptyTargetViolationError + unsupportedV *UnsupportedVersionError + drainVerify *DrainVerificationError + configErr *ConfigError + prefixErr *PrefixConflictError + ) + switch { + case errors.As(err, &unsupportedV): + return "UnsupportedVersion" + case errors.As(err, &cpInvalid): + return "CheckpointInvalid" + case errors.As(err, &emptyTarget): + return "EmptyTargetViolation" + case errors.As(err, &prefixErr): + return "PrefixConflict" + case errors.As(err, &drainVerify): + return "DrainVerificationFailed" + case errors.As(err, &tooLarge): + return "RequestTooLarge" + case errors.As(err, &configErr): + return "InvalidConfig" + case errors.As(err, &fenceErr): + return "FenceLost" + } + return "" +} + +// Classify maps any error the engine encounters to its taxonomy class. +// Typed engine errors win; then etcd's typed rpc errors; then gRPC status +// codes; unknown errors default to transient (retrying an unknown error is +// recoverable, silently dropping a permanent one is not). +// +// The engine owns 100% of write-path retry per the class returned here — +// see the retry-ownership contract on [backoff]. Never classify by gRPC +// code alone: ErrNoSpace (quota), ErrTooManyRequests (throttle), and the +// client send cap (permanent) all share codes.ResourceExhausted. +func Classify(err error) Class { + if err == nil { + return ClassTransient + } + + // Engine-typed errors first. + var ( + resyncErr *ResyncError + cpInvalid *CheckpointInvalidError + fenceErr *FenceError + tooLarge *TooLargeError + emptyTarget *EmptyTargetViolationError + unsupportedV *UnsupportedVersionError + drainVerify *DrainVerificationError + configErr *ConfigError + prefixErr *PrefixConflictError + ) + switch { + case errors.As(err, &resyncErr): + return ClassResync + case errors.As(err, &cpInvalid), + errors.As(err, &fenceErr), + errors.As(err, &tooLarge), + errors.As(err, &emptyTarget), + errors.As(err, &unsupportedV), + errors.As(err, &drainVerify), + errors.As(err, &configErr), + errors.As(err, &prefixErr): + return ClassPermanent + } + + // etcd-typed errors: normalize the raw gRPC error to its canonical + // rpctypes singleton where one exists, so both wire and pre-converted + // forms classify identically. + switch rpctypes.Error(err) { + case rpctypes.ErrNoSpace: + return ClassQuota + case rpctypes.ErrTooManyRequests: + return ClassThrottle + case rpctypes.ErrCompacted, rpctypes.ErrFutureRev: + return ClassResync + case rpctypes.ErrTooManyOps, rpctypes.ErrRequestTooLarge: + return ClassPermanent + case rpctypes.ErrPermissionDenied, rpctypes.ErrUserEmpty, rpctypes.ErrAuthFailed: + return ClassPermanent + } + + // Context errors: cancellation/deadline of our own contexts. + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return ClassTransient + } + + // Remaining gRPC status codes. + if s, ok := status.FromError(err); ok { + switch s.Code() { + case codes.ResourceExhausted: + // The client-side send cap surfaces as ResourceExhausted + // "trying to send message larger than max": that is an oversized + // request, NOT throttling — mislabelling it throttling retries a + // poison batch forever. + if strings.Contains(s.Message(), "larger than max") { + return ClassPermanent + } + return ClassThrottle + case codes.InvalidArgument: + return ClassPermanent + case codes.PermissionDenied, codes.Unauthenticated: + return ClassPermanent + case codes.OutOfRange: + return ClassResync + } + } + + return ClassTransient +} diff --git a/pkg/mirroragent/errors_test.go b/pkg/mirroragent/errors_test.go new file mode 100644 index 00000000..f9bc155e --- /dev/null +++ b/pkg/mirroragent/errors_test.go @@ -0,0 +1,167 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" +) + +func TestClassify(t *testing.T) { + cases := []struct { + name string + err error + want Class + }{ + // Engine-typed errors. + {name: "resync compacted", err: &ResyncError{Reason: ResyncReasonCompacted}, want: ClassResync}, + {name: "resync cluster id", err: &ResyncError{Reason: ResyncReasonClusterIDMismatch}, want: ClassResync}, + {name: "wrapped resync", err: fmt.Errorf("cycle: %w", &ResyncError{Reason: ResyncReasonCompacted}), + want: ClassResync}, + {name: "checkpoint invalid fails closed", err: &CheckpointInvalidError{Reason: "garbage"}, + want: ClassPermanent}, + {name: "fence violation", err: &FenceError{Detail: "taken over"}, want: ClassPermanent}, + {name: "too large", err: &TooLargeError{Key: "/dst/…deadbeef"}, want: ClassPermanent}, + {name: "empty target violation", err: &EmptyTargetViolationError{KeyCount: 3}, want: ClassPermanent}, + {name: "unsupported version", err: &UnsupportedVersionError{Side: "source", Version: "3.3.0"}, + want: ClassPermanent}, + {name: "drain verification", err: &DrainVerificationError{SourceKeys: 1, TargetKeys: 2}, + want: ClassPermanent}, + {name: "config error", err: &ConfigError{Detail: "no endpoints"}, want: ClassPermanent}, + {name: "prefix conflict", err: &PrefixConflictError{Key: "/dst/…deadbeef", OwnerLinkUID: "other"}, + want: ClassPermanent}, + + // etcd-typed rpc errors (client-side singletons). + {name: "no space", err: rpctypes.ErrNoSpace, want: ClassQuota}, + {name: "too many requests", err: rpctypes.ErrTooManyRequests, want: ClassThrottle}, + {name: "compacted", err: rpctypes.ErrCompacted, want: ClassResync}, + {name: "future rev", err: rpctypes.ErrFutureRev, want: ClassResync}, + {name: "too many ops", err: rpctypes.ErrTooManyOps, want: ClassPermanent}, + {name: "request too large", err: rpctypes.ErrRequestTooLarge, want: ClassPermanent}, + {name: "permission denied", err: rpctypes.ErrPermissionDenied, want: ClassPermanent}, + + // etcd-typed rpc errors (gRPC wire form). + {name: "no space wire", err: rpctypes.ErrGRPCNoSpace, want: ClassQuota}, + {name: "compacted wire", err: rpctypes.ErrGRPCCompacted, want: ClassResync}, + + // The three-way codes.ResourceExhausted disambiguation: identical + // gRPC code, three different classes — classification must never be + // by code alone. + {name: "resource exhausted quota", + err: status.Error(codes.ResourceExhausted, "etcdserver: mvcc: database space exceeded"), + want: ClassQuota}, + {name: "resource exhausted throttle", + err: status.Error(codes.ResourceExhausted, "etcdserver: too many requests"), + want: ClassThrottle}, + {name: "resource exhausted client send cap", + err: status.Error(codes.ResourceExhausted, + "trying to send message larger than max (3145728 vs. 2097152)"), + want: ClassPermanent}, + {name: "resource exhausted unknown rate flavor", + err: status.Error(codes.ResourceExhausted, "some proxy rate limit"), + want: ClassThrottle}, + + // Context and transport errors. + {name: "deadline exceeded", err: context.DeadlineExceeded, want: ClassTransient}, + {name: "canceled", err: context.Canceled, want: ClassTransient}, + {name: "no leader", err: rpctypes.ErrNoLeader, want: ClassTransient}, + {name: "unavailable", err: status.Error(codes.Unavailable, "connection refused"), want: ClassTransient}, + {name: "invalid argument", err: status.Error(codes.InvalidArgument, "etcdserver: request is too large"), + want: ClassPermanent}, + {name: "unauthenticated", err: status.Error(codes.Unauthenticated, "invalid auth token"), + want: ClassPermanent}, + {name: "out of range", err: status.Error(codes.OutOfRange, "required revision has been compacted"), + want: ClassResync}, + + // Unknowns default to transient: retrying an unknown error is + // recoverable, silently dropping a permanent one is not. + {name: "unknown error", err: errors.New("weather is bad"), want: ClassTransient}, + {name: "nil", err: nil, want: ClassTransient}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, Classify(tc.err)) + }) + } +} + +func TestReasonFor(t *testing.T) { + cases := []struct { + name string + err error + want string + }{ + {name: "unsupported version", err: &UnsupportedVersionError{Side: "source", Version: "3.3.0"}, + want: "UnsupportedVersion"}, + {name: "checkpoint invalid", err: &CheckpointInvalidError{Reason: "garbage"}, + want: "CheckpointInvalid"}, + {name: "empty target violation", err: &EmptyTargetViolationError{KeyCount: 3}, + want: "EmptyTargetViolation"}, + {name: "prefix conflict", err: &PrefixConflictError{Key: "/dst/…deadbeef", OwnerLinkUID: "other"}, + want: "PrefixConflict"}, + {name: "drain verification", err: &DrainVerificationError{SourceKeys: 1, TargetKeys: 2}, + want: "DrainVerificationFailed"}, + {name: "too large", err: &TooLargeError{Key: "/dst/…deadbeef"}, want: "RequestTooLarge"}, + {name: "config error", err: &ConfigError{Detail: "no endpoints"}, want: "InvalidConfig"}, + {name: "fence violation", err: &FenceError{Detail: "taken over"}, want: "FenceLost"}, + {name: "wrapped typed error", err: fmt.Errorf("probing source: %w", + &UnsupportedVersionError{Side: "source", Version: "3.2.0"}), want: "UnsupportedVersion"}, + {name: "plain error", err: errors.New("weather is bad"), want: ""}, + {name: "etcd rpc error", err: rpctypes.ErrNoSpace, want: ""}, + {name: "nil", err: nil, want: ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, ReasonFor(tc.err)) + }) + } +} + +func TestRedactKey(t *testing.T) { + secret := []byte("/dst/tenants/acme/api-token-primary") + got := RedactKey("/dst/", secret) + + assert.Equal(t, got, RedactKey("/dst/", secret), "redaction must be deterministic") + assert.True(t, strings.HasPrefix(got, "/dst/…"), "the public prefix survives: %q", got) + assert.Len(t, got, len("/dst/…")+8, "prefix + ellipsis + 8 hex chars") + assert.NotContains(t, got, "tenants", "no raw key bytes beyond the prefix may surface") + assert.NotContains(t, got, "acme") + assert.NotEqual(t, RedactKey("/dst/", []byte("/dst/other")), got) +} + +func TestTooLargeErrorNeverLeaksRawKey(t *testing.T) { + // Mirrors the construction sites: Key is always the RedactKey form. + e := &TooLargeError{ + Key: RedactKey("/dst/", []byte("/dst/tenants/acme/api-token-primary")), + Ops: 2, + Bytes: 3 << 20, + Cause: errors.New("etcdserver: request is too large"), + } + msg := e.Error() + assert.NotContains(t, msg, "acme", "error text must not carry raw key bytes") + assert.Contains(t, msg, "/dst/…", "error text must carry the redacted key for operators") + assert.Equal(t, ClassPermanent, Classify(e)) +} diff --git a/pkg/mirroragent/fence.go b/pkg/mirroragent/fence.go new file mode 100644 index 00000000..ee672a7d --- /dev/null +++ b/pkg/mirroragent/fence.go @@ -0,0 +1,150 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "encoding/json" + "fmt" +) + +// FenceRole is the role stamped into the fence key. Applies are only legal +// while the role is Mirror; the drain flow flips it to Primary at cutover so +// a straggler mirror apply fails its mod-revision compare loudly. +type FenceRole string + +const ( + // RoleMirror means the mirror owns the destination prefix and applies + // are in flight. + RoleMirror FenceRole = "Mirror" + // RolePrimary means cutover completed: the destination prefix is now + // authoritative and no mirror may write under it. + RolePrimary FenceRole = "Primary" +) + +// FenceVersion is the current checkpoint wire-format version. Decoding fails +// closed on any other version (distinct from an absent checkpoint). +const FenceVersion = 1 + +// FenceValue is the checkpoint/fence document stored at the reserved key in +// the target etcd. It is written in the same Txn as every applied batch; +// its mod revision is the compare every write path is fenced on. +// +// State encoding — the tuple (Scanning, PrunePending, Role) subsumes an +// explicit phase enum: +// +// Scanning=true genesis scan in flight: Watermark is the +// scan's watch-start revision R0 (the replay +// base), NOT a caught-up-through claim; +// progress lives in ScanCursor/SubRevision. +// Consumers must never read Watermark as +// replication progress while Scanning is true. +// Scanning=false, Role=Mirror steady state: Watermark is the source +// revision fully applied through. +// PrunePending=true a forced resync's mandatory mark-and-sweep +// is owed; survives crashes. +// Role=Primary cutover complete: no mirror may write under +// the destination prefix; stragglers fail +// their mod-revision compare loudly. +type FenceValue struct { + // Version is the wire-format version (FenceVersion). + Version int `json:"v"` + // LinkUID identifies the mirror link that owns this fence. + LinkUID string `json:"linkUID"` + // Epoch is the agent generation that last wrote the checkpoint. + Epoch int64 `json:"epoch"` + // Role is Mirror until cutover flips it to Primary. + Role FenceRole `json:"role"` + // Watermark is the source revision through which the target is caught + // up. While a genesis scan is in flight it is the scan's watch-start + // revision (the base the buffered watch replays over), not a fully + // applied revision — Scanning distinguishes the two. + Watermark int64 `json:"watermark"` + // SubRevision is the ordinal progress marker within a Watermark that is + // not yet revision-complete: the genesis scan stamps its page ordinal + // here. Zero on every revision-complete checkpoint. + SubRevision int64 `json:"subRevision,omitempty"` + // Scanning is true while a genesis scan is in flight; ScanCursor is then + // the last source key whose page has been applied, so a restarted agent + // resumes the scan instead of starting over. + Scanning bool `json:"scanning,omitempty"` + ScanCursor string `json:"scanCursor,omitempty"` + // PrunePending is true from the moment a forced resync claims the fence + // until its mandatory mark-and-sweep prune pass has completed. It makes + // the owed sweep durable: an agent that crashes mid-forced-resync and + // resumes the scan still runs the prune, so deletes from the blind window + // that triggered the resync cannot silently resurrect on the target. + PrunePending bool `json:"prunePending,omitempty"` + // SourceClusterID / TargetClusterID bind the checkpoint to BOTH cluster + // identities. Either changing means an endpoint now points at a + // different cluster than the checkpoint was taken against: the + // checkpoint is invalidated, genesis is forced, and RequireEmpty + // re-arms. String-encoded: cluster IDs use the full uint64 range. + SourceClusterID uint64 `json:"sourceClusterID,string"` + TargetClusterID uint64 `json:"targetClusterID,string"` +} + +// Encode serializes the fence value for storage at the reserved key. The +// wire-format version is stamped unconditionally. +func (f FenceValue) Encode() (string, error) { + f.Version = FenceVersion + if err := f.validate(); err != nil { + return "", err + } + b, err := json.Marshal(f) + if err != nil { + return "", err + } + return string(b), nil +} + +// DecodeFenceValue parses a stored checkpoint. Corrupt content or an unknown +// version returns a *CheckpointInvalidError, which classifies PERMANENT: an +// undecodable fence proves nothing about ownership, epoch ordering, or role +// (it may be a newer agent generation's format, or a corrupted post-cutover +// Primary fence), so no write — least of all a resync's prune — is provably +// safe. The operator must inspect and delete the reserved key to recover. +func DecodeFenceValue(raw []byte) (FenceValue, error) { + var f FenceValue + if err := json.Unmarshal(raw, &f); err != nil { + return FenceValue{}, &CheckpointInvalidError{Reason: fmt.Sprintf("undecodable checkpoint: %v", err)} + } + if f.Version != FenceVersion { + return FenceValue{}, &CheckpointInvalidError{ + Reason: fmt.Sprintf("unknown checkpoint version %d (agent supports %d)", f.Version, FenceVersion), + } + } + if err := f.validate(); err != nil { + return FenceValue{}, &CheckpointInvalidError{Reason: err.Error()} + } + return f, nil +} + +func (f FenceValue) validate() error { + if f.LinkUID == "" { + return fmt.Errorf("checkpoint has empty linkUID") + } + if f.Epoch < 1 { + return fmt.Errorf("checkpoint has invalid epoch %d", f.Epoch) + } + if f.Role != RoleMirror && f.Role != RolePrimary { + return fmt.Errorf("checkpoint has invalid role %q", f.Role) + } + if f.Watermark < 0 || f.SubRevision < 0 { + return fmt.Errorf("checkpoint has negative revision fields") + } + return nil +} diff --git a/pkg/mirroragent/fence_test.go b/pkg/mirroragent/fence_test.go new file mode 100644 index 00000000..be53db29 --- /dev/null +++ b/pkg/mirroragent/fence_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func validFence() FenceValue { + return FenceValue{ + LinkUID: "link-1", + Epoch: 3, + Role: RoleMirror, + Watermark: 42, + SubRevision: 7, + Scanning: true, + ScanCursor: "/src/key-0100", + PrunePending: true, + SourceClusterID: 0xdeadbeefcafef00d, + TargetClusterID: 0x0123456789abcdef, + } +} + +func TestFenceValueRoundTrip(t *testing.T) { + in := validFence() + raw, err := in.Encode() + require.NoError(t, err) + + out, err := DecodeFenceValue([]byte(raw)) + require.NoError(t, err) + in.Version = FenceVersion // Encode stamps it + assert.Equal(t, in, out) + assert.True(t, out.PrunePending, "PrunePending must survive the round trip") + assert.Equal(t, uint64(0xdeadbeefcafef00d), out.SourceClusterID, + "cluster IDs must survive the full uint64 range (string-encoded)") +} + +// TestDecodeFenceValueFailsClosed pins the WIP semantics: corrupt content or +// an unknown wire version is a *CheckpointInvalidError AND classifies +// Permanent — never a resync, never a resume on a guess. +func TestDecodeFenceValueFailsClosed(t *testing.T) { + cases := []struct { + name string + raw string + }{ + {name: "garbage", raw: "not json at all {"}, + {name: "empty", raw: ""}, + {name: "future version", raw: `{"v":99,"linkUID":"l","epoch":1,"role":"Mirror",` + + `"watermark":1,"sourceClusterID":"1","targetClusterID":"2"}`}, + {name: "version zero", raw: `{"linkUID":"l","epoch":1,"role":"Mirror",` + + `"sourceClusterID":"1","targetClusterID":"2"}`}, + {name: "empty linkUID", raw: `{"v":1,"linkUID":"","epoch":1,"role":"Mirror",` + + `"sourceClusterID":"1","targetClusterID":"2"}`}, + {name: "epoch below one", raw: `{"v":1,"linkUID":"l","epoch":0,"role":"Mirror",` + + `"sourceClusterID":"1","targetClusterID":"2"}`}, + {name: "unknown role", raw: `{"v":1,"linkUID":"l","epoch":1,"role":"Standby",` + + `"sourceClusterID":"1","targetClusterID":"2"}`}, + {name: "negative watermark", raw: `{"v":1,"linkUID":"l","epoch":1,"role":"Mirror",` + + `"watermark":-5,"sourceClusterID":"1","targetClusterID":"2"}`}, + {name: "negative subrevision", raw: `{"v":1,"linkUID":"l","epoch":1,"role":"Mirror",` + + `"subRevision":-1,"sourceClusterID":"1","targetClusterID":"2"}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := DecodeFenceValue([]byte(tc.raw)) + var ci *CheckpointInvalidError + require.ErrorAs(t, err, &ci) + assert.Equal(t, ClassPermanent, Classify(err), + "an undecodable checkpoint must fail closed (permanent), not resync") + }) + } +} + +func TestFenceValueEncodeRejectsInvalid(t *testing.T) { + cases := []struct { + name string + mutate func(*FenceValue) + }{ + {name: "empty linkUID", mutate: func(f *FenceValue) { f.LinkUID = "" }}, + {name: "epoch below one", mutate: func(f *FenceValue) { f.Epoch = 0 }}, + {name: "bad role", mutate: func(f *FenceValue) { f.Role = "Replica" }}, + {name: "negative watermark", mutate: func(f *FenceValue) { f.Watermark = -1 }}, + {name: "negative subrevision", mutate: func(f *FenceValue) { f.SubRevision = -2 }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := validFence() + tc.mutate(&f) + _, err := f.Encode() + assert.Error(t, err) + }) + } +} diff --git a/pkg/mirroragent/helpers_integration_test.go b/pkg/mirroragent/helpers_integration_test.go new file mode 100644 index 00000000..919d4841 --- /dev/null +++ b/pkg/mirroragent/helpers_integration_test.go @@ -0,0 +1,305 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent_test + +import ( + "context" + "fmt" + "net" + "net/url" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd-operator/pkg/mirroragent" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" +) + +// startEtcd boots one in-process embedded etcd on unique loopback ports and +// returns a client wired the way the engine requires (NewClientConfig). +func startEtcd(t *testing.T, mutate func(*embed.Config)) *clientv3.Client { + t.Helper() + cfg := embed.NewConfig() + cfg.Dir = t.TempDir() + cfg.Logger = "zap" + cfg.LogLevel = "error" + cu := freeURL(t) + pu := freeURL(t) + cfg.ListenClientUrls = []url.URL{*cu} + cfg.AdvertiseClientUrls = []url.URL{*cu} + cfg.ListenPeerUrls = []url.URL{*pu} + cfg.AdvertisePeerUrls = []url.URL{*pu} + cfg.InitialCluster = cfg.Name + "=" + pu.String() + if mutate != nil { + mutate(cfg) + } + e, err := embed.StartEtcd(cfg) + require.NoError(t, err) + t.Cleanup(e.Close) + select { + case <-e.Server.ReadyNotify(): + case <-time.After(60 * time.Second): + t.Fatal("embedded etcd took too long to start") + } + cli, err := clientv3.New(mirroragent.NewClientConfig([]string{cu.String()}, nil, 5*time.Second)) + require.NoError(t, err) + t.Cleanup(func() { _ = cli.Close() }) + return cli +} + +func freeURL(t *testing.T) *url.URL { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + addr := l.Addr().String() + require.NoError(t, l.Close()) + u, err := url.Parse("http://" + addr) + require.NoError(t, err) + return u +} + +// baseCfg returns an engine config with intervals shrunk for test runtime. +func baseCfg(srcPrefix, dstPrefix string) mirroragent.Config { + return mirroragent.Config{ + LinkUID: "test-link", + Epoch: 1, + SourcePrefix: srcPrefix, + TargetPrefix: dstPrefix, + RequestTimeout: 5 * time.Second, + BackoffInitialDelay: 50 * time.Millisecond, + BackoffMaxDelay: 500 * time.Millisecond, + ProgressInterval: 200 * time.Millisecond, + QuotaProbeInterval: 250 * time.Millisecond, + } +} + +func checkpointKey(cfg mirroragent.Config) string { + return cfg.TargetPrefix + cfg.DestPrefix + mirroragent.DefaultCheckpointKeySuffix +} + +type agentRun struct { + agent *mirroragent.Agent + cancel context.CancelFunc + done chan error +} + +// startAgent builds and runs an Agent in a goroutine, cleaned up with the +// test. +func startAgent(t *testing.T, cfg mirroragent.Config, src, dst mirroragent.Client) *agentRun { + t.Helper() + agent, err := mirroragent.New(cfg, src, dst) + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- agent.Run(ctx) }() + r := &agentRun{agent: agent, cancel: cancel, done: done} + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(15 * time.Second): + t.Error("agent did not stop within 15s of cancel") + } + }) + return r +} + +// stop cancels the agent and waits for Run to return. +func (r *agentRun) stop(t *testing.T) error { + t.Helper() + r.cancel() + return r.waitErr(t, 15*time.Second) +} + +// waitErr waits for Run to return and hands back its error. +func (r *agentRun) waitErr(t *testing.T, timeout time.Duration) error { + t.Helper() + select { + case err := <-r.done: + r.done <- err // allow repeated reads / cleanup + return err + case <-time.After(timeout): + t.Fatalf("agent Run did not return within %v", timeout) + return nil + } +} + +// waitSnap polls Snapshot until cond holds. +func waitSnap( + t *testing.T, a *mirroragent.Agent, timeout time.Duration, + what string, cond func(mirroragent.Snapshot) bool, +) mirroragent.Snapshot { + t.Helper() + deadline := time.Now().Add(timeout) + var s mirroragent.Snapshot + for time.Now().Before(deadline) { + s = a.Snapshot() + if cond(s) { + return s + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("waiting for %s: condition not met within %v; last snapshot: %+v", what, timeout, s) + return s +} + +// putN writes n sequential keys under prefix and returns the resulting +// source data as the expected target map under dstPrefix. +func putN(t *testing.T, cli *clientv3.Client, srcPrefix, dstPrefix string, n int) map[string]string { + t.Helper() + want := make(map[string]string, n) + for i := range n { + k := fmt.Sprintf("key-%04d", i) + v := fmt.Sprintf("val-%04d", i) + _, err := cli.Put(t.Context(), srcPrefix+k, v) + require.NoError(t, err) + want[dstPrefix+k] = v + } + return want +} + +// targetData reads every key under the destination prefix except the +// reserved checkpoint key. +func targetData(t *testing.T, cli *clientv3.Client, cfg mirroragent.Config) map[string]string { + t.Helper() + resp, err := cli.Get(t.Context(), cfg.TargetPrefix+cfg.DestPrefix, clientv3.WithPrefix()) + require.NoError(t, err) + out := make(map[string]string, len(resp.Kvs)) + for _, kv := range resp.Kvs { + if string(kv.Key) == checkpointKey(cfg) { + continue + } + out[string(kv.Key)] = string(kv.Value) + } + return out +} + +// waitTargetData polls until the destination data (reserved key excluded) +// equals want exactly. +func waitTargetData( + t *testing.T, cli *clientv3.Client, cfg mirroragent.Config, + timeout time.Duration, want map[string]string, +) { + t.Helper() + deadline := time.Now().Add(timeout) + var got map[string]string + for time.Now().Before(deadline) { + got = targetData(t, cli, cfg) + if mapsEqual(got, want) { + return + } + time.Sleep(25 * time.Millisecond) + } + t.Fatalf("target data never converged: got %d keys, want %d keys\ngot: %v\nwant: %v", + len(got), len(want), summarize(got), summarize(want)) +} + +func mapsEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +func summarize(m map[string]string) string { + if len(m) <= 12 { + return fmt.Sprintf("%v", m) + } + keys := make([]string, 0, 12) + for k := range m { + keys = append(keys, k) + if len(keys) == 12 { + break + } + } + return fmt.Sprintf("{%s, ...}", strings.Join(keys, ", ")) +} + +// readFence reads and decodes the reserved checkpoint key. +func readFence( + t *testing.T, cli *clientv3.Client, cfg mirroragent.Config, +) (mirroragent.FenceValue, int64) { + t.Helper() + resp, err := cli.Get(t.Context(), checkpointKey(cfg)) + require.NoError(t, err) + require.Len(t, resp.Kvs, 1, "reserved checkpoint key missing") + f, err := mirroragent.DecodeFenceValue(resp.Kvs[0].Value) + require.NoError(t, err) + return f, resp.Kvs[0].ModRevision +} + +// sourceRevision returns the source cluster's current revision. +func sourceRevision(t *testing.T, cli *clientv3.Client, prefix string) int64 { + t.Helper() + resp, err := cli.Get(t.Context(), prefix, clientv3.WithPrefix(), clientv3.WithCountOnly()) + require.NoError(t, err) + return resp.Header.Revision +} + +// countingClient wraps a Client and counts Get/Txn calls, for asserting that +// resume paths do not rescan and quota parking does not hot-loop. +type countingClient struct { + mirroragent.Client + gets atomic.Int64 + txns atomic.Int64 +} + +func (c *countingClient) Get( + ctx context.Context, key string, opts ...clientv3.OpOption, +) (*clientv3.GetResponse, error) { + c.gets.Add(1) + return c.Client.Get(ctx, key, opts...) +} + +func (c *countingClient) Txn(ctx context.Context) clientv3.Txn { + c.txns.Add(1) + return c.Client.Txn(ctx) +} + +// watchRevRecordingClient records the OpOption-resolved start revision of +// every Watch call, for pinning resume revisions. +type watchRevRecordingClient struct { + mirroragent.Client + mu sync.Mutex + revs []int64 +} + +func (c *watchRevRecordingClient) Watch( + ctx context.Context, key string, opts ...clientv3.OpOption, +) clientv3.WatchChan { + op := clientv3.OpGet(key, opts...) + c.mu.Lock() + c.revs = append(c.revs, op.Rev()) + c.mu.Unlock() + return c.Client.Watch(ctx, key, opts...) +} + +func (c *watchRevRecordingClient) recorded() []int64 { + c.mu.Lock() + defer c.mu.Unlock() + return append([]int64(nil), c.revs...) +} diff --git a/pkg/mirroragent/integration_delta_test.go b/pkg/mirroragent/integration_delta_test.go new file mode 100644 index 00000000..00b9283b --- /dev/null +++ b/pkg/mirroragent/integration_delta_test.go @@ -0,0 +1,1297 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Integration tests for the Design-3 gap-closure delta: replay-buffer +// overflow restarts, the durable PrunePending flag, cluster-ID re-arm, +// the resync-loop latch, exclude-range elision, oversize permanence, +// fail-closed checkpoints, the version floor, progress-notify liveness, +// and duplicate-replay idempotence. +package mirroragent_test + +import ( + "context" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "go.etcd.io/etcd-operator/pkg/mirroragent" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" +) + +// bigValueClient returns a client whose send cap admits values larger than +// the 2MiB clientv3 default, for seeding oversize test data. +func bigValueClient(t *testing.T, target *clientv3.Client) *clientv3.Client { + t.Helper() + cfg := mirroragent.NewClientConfig(target.Endpoints(), nil, 5*time.Second) + cfg.MaxCallSendMsgSize = 16 << 20 + cli, err := clientv3.New(cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = cli.Close() }) + return cli +} + +// TestWatchBufferOverflowRestart covers the bounded replay buffer: with a +// tiny WatchBufferBytes and sustained mid-scan churn, genesis attempts are +// aborted and restarted from a fresh R0 (never unbounded growth), the cause +// is surfaced in the snapshot, repeated restarts trip the resync-loop +// detector, and once churn stops the mirror still converges byte-exact. +func TestWatchBufferOverflowRestart(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.WatchBufferBytes = 512 // a handful of events + cfg.MaxOpsPerSecond = 150 // keep each scan attempt slow enough to race + cfg.PageKeyLimit = 25 + cfg.ResyncLoopThreshold = 2 + + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 150) + + r := startAgent(t, cfg, src, dst) + + // Churn during the scan: each write lands in the replay buffer. Values + // are sized so a SINGLE churn event overflows the 512-byte buffer — + // overflow must not depend on sustaining a wall-clock churn rate on a + // loaded runner. + churnValue := strings.Repeat("c", 1024) + churnCtx, stopChurn := context.WithCancel(t.Context()) + defer stopChurn() + churnDone := make(chan struct{}) + go func() { + defer close(churnDone) + for i := 0; churnCtx.Err() == nil; i++ { + k := fmt.Sprintf("churn-%04d", i) + if _, err := src.Put(churnCtx, cfg.SourcePrefix+k, churnValue); err != nil { + return + } + time.Sleep(5 * time.Millisecond) + } + }() + + snap := waitSnap(t, r.agent, 30*time.Second, "buffer overflow restarts", + func(s mirroragent.Snapshot) bool { return s.ScanRestartCount >= 2 }) + assert.Equal(t, mirroragent.ScanRestartWatchBufferOverflow, snap.LastScanRestartCause) + assert.EqualValues(t, 0, snap.ForcedResyncCount, + "a buffer-bound restart is not a forced resync — the checkpoint was never invalidated") + waitSnap(t, r.agent, 30*time.Second, "resync-loop detector tripped by repeated overflows", + func(s mirroragent.Snapshot) bool { return s.ResyncLoopDetected }) + + stopChurn() + <-churnDone + + // Byte-exact convergence against source truth, re-read each poll: a + // churn Put cancelled mid-flight can still land server-side after the + // stop signal, so the authority is whatever the source holds NOW. + var want map[string]string + deadline := time.Now().Add(90 * time.Second) + for { + sresp, err := src.Get(t.Context(), cfg.SourcePrefix, clientv3.WithPrefix()) + require.NoError(t, err) + want = make(map[string]string, len(sresp.Kvs)) + for _, kv := range sresp.Kvs { + want[cfg.TargetPrefix+strings.TrimPrefix(string(kv.Key), cfg.SourcePrefix)] = string(kv.Value) + } + if got := targetData(t, dst, cfg); mapsEqual(got, want) { + break + } + require.True(t, time.Now().Before(deadline), + "target never converged to source truth after churn stopped") + time.Sleep(100 * time.Millisecond) + } + + // Post-churn writes reach steady state and clear the detector. A write + // can still be swallowed into a final scan/replay (which must NOT clear + // the latch), so keep writing until one lands as a live tail apply. + deadline = time.Now().Add(30 * time.Second) + for i := 0; r.agent.Snapshot().ResyncLoopDetected; i++ { + require.True(t, time.Now().Before(deadline), + "detector never cleared at steady state despite live writes") + _, err := src.Put(t.Context(), fmt.Sprintf("/src/settled-%d", i), "yes") + require.NoError(t, err) + time.Sleep(150 * time.Millisecond) + } +} + +// TestPrunePendingCrashResume is THE new-mechanism test: an agent that is +// killed mid-forced-resync — after the fence records PrunePending=true but +// before the mark-and-sweep ran — must still run the prune after restart, +// so a delete from the blind window cannot resurrect on the target. +func TestPrunePendingCrashResume(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 40) + r1 := startAgent(t, cfg, src, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + waitSnap(t, r1.agent, 10*time.Second, "run 1 Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + require.ErrorIs(t, r1.stop(t), context.Canceled) + + // Blind window while the agent is down: delete one mirrored key, add + // two, then compact past the watermark so the re-watch is doomed. + ctx := t.Context() + _, err := src.Delete(ctx, "/src/key-0005") + require.NoError(t, err) + delete(want, "/dst/key-0005") + for i := range 2 { + k := fmt.Sprintf("late-%d", i) + _, perr := src.Put(ctx, cfg.SourcePrefix+k, "late") + require.NoError(t, perr) + want[cfg.TargetPrefix+k] = "late" + } + head, err := src.Get(ctx, "/src/", clientv3.WithPrefix(), clientv3.WithCountOnly()) + require.NoError(t, err) + _, err = src.Compact(ctx, head.Header.Revision, clientv3.WithCompactPhysical()) + require.NoError(t, err) + + // Run 2 hits ErrCompacted and starts the forced resync (slowed down and + // with small Txns so the fence records a MID-SCAN cursor). It is killed + // only once the fence shows PrunePending AND a non-empty scan cursor, so + // run 3 must exercise the cursor-resume branch, not the fresh-R0 path. + cfg2 := cfg + cfg2.Epoch = 2 + cfg2.MaxOpsPerSecond = 25 + cfg2.MaxTxnOps = 6 // 5 data slots: the cursor advances every 5 keys + r2 := startAgent(t, cfg2, src, dst) + deadline := time.Now().Add(20 * time.Second) + for { + require.True(t, time.Now().Before(deadline), + "fence never recorded PrunePending with a mid-scan cursor") + resp, gerr := dst.Get(ctx, checkpointKey(cfg)) + require.NoError(t, gerr) + if len(resp.Kvs) == 1 { + f, derr := mirroragent.DecodeFenceValue(resp.Kvs[0].Value) + require.NoError(t, derr) + if f.PrunePending && f.Scanning && f.ScanCursor != "" { + break + } + } + time.Sleep(5 * time.Millisecond) + } + r2.cancel() + require.ErrorIs(t, r2.waitErr(t, 15*time.Second), context.Canceled) + + // The kill must have landed mid-scan for the resume branch to be under + // test at all; the paced scan (40 keys at 25 ops/s) makes this window + // seconds wide. + killed, _ := readFence(t, dst, cfg) + require.True(t, killed.Scanning && killed.ScanCursor != "", + "test premise: run 2 must die MID-SCAN (fence: %+v) — widen the pacing if this trips", killed) + + // The blind-window delete has resurrection potential right now. + resp, err := dst.Get(ctx, "/dst/key-0005") + require.NoError(t, err) + require.Len(t, resp.Kvs, 1, + "test setup: the deleted key must still be on the target before the prune") + + // Run 3 resumes purely from the durable fence state: the scan continues + // from the recorded cursor (no re-count, no re-applied pre-cursor keys) + // and the owed sweep still runs without re-detecting the compaction. + cfg3 := cfg + cfg3.Epoch = 3 + rsrc := &rangeRecordingClient{Client: src} + r3 := startAgent(t, cfg3, rsrc, dst) + waitTargetData(t, dst, cfg, 30*time.Second, want) + snap := waitSnap(t, r3.agent, 20*time.Second, "run 3 Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + assert.EqualValues(t, 0, snap.ForcedResyncCount, + "run 3 resumed an owed prune; it did not have to re-detect the compaction") + f, _ := readFence(t, dst, cfg) + assert.False(t, f.PrunePending, "the flag clears once the sweep completed") + assert.False(t, f.Scanning) + + // Resume-branch pin: run 3's FIRST source read must be the scan page + // starting just after the recorded cursor. The fresh-R0 path would open + // with a CountOnly read at the range start instead — and a broken resume + // (panic/skip) could only be reached through that first read. + windows := rsrc.recorded() + require.NotEmpty(t, windows) + assert.Equal(t, killed.ScanCursor+"\x00", windows[0][0], + "run 3's first source read must resume at nextKey(fence.ScanCursor), not rescan from the start") +} + +// TestClusterIDMismatchRearm covers the dual-identity binding: a checkpoint +// bound to a different source OR target cluster forces genesis and RE-ARMS +// RequireEmpty (unlike an ordinary forced resync, which skips it because the +// fence proves ownership). +func TestClusterIDMismatchRearm(t *testing.T) { + t.Run("FreshTargetRearms", func(t *testing.T) { + src := startEtcd(t, nil) + dstA := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 5) + r1 := startAgent(t, cfg, src, dstA) + waitTargetData(t, dstA, cfg, 20*time.Second, want) + require.ErrorIs(t, r1.stop(t), context.Canceled) + + // A rebuilt target carrying the old fence (e.g. restored from a + // snapshot of another cluster) plus pre-existing data. + dstB := startEtcd(t, nil) + ctx := t.Context() + fresp, err := dstA.Get(ctx, checkpointKey(cfg)) + require.NoError(t, err) + require.Len(t, fresp.Kvs, 1) + staleFence := string(fresp.Kvs[0].Value) + _, err = dstB.Put(ctx, checkpointKey(cfg), staleFence) + require.NoError(t, err) + _, err = dstB.Put(ctx, "/dst/preexisting", "dirty") + require.NoError(t, err) + + cfg2 := cfg + cfg2.Epoch = 2 + r2 := startAgent(t, cfg2, src, dstB) + runErr := r2.waitErr(t, 20*time.Second) + var ev *mirroragent.EmptyTargetViolationError + require.ErrorAs(t, runErr, &ev, + "the re-armed RequireEmpty must trip on the non-empty fresh target, got: %v", runErr) + snap := r2.agent.Snapshot() + assert.Equal(t, mirroragent.ResyncReasonClusterIDMismatch, snap.LastResyncReason) + assert.EqualValues(t, 1, snap.ForcedResyncCount) + + // Failing the gate must write nothing — the stale fence is intact. + after, err := dstB.Get(ctx, checkpointKey(cfg)) + require.NoError(t, err) + require.Len(t, after.Kvs, 1) + assert.Equal(t, staleFence, string(after.Kvs[0].Value)) + }) + + t.Run("FreshSourceRearms", func(t *testing.T) { + srcA := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, srcA, cfg.SourcePrefix, cfg.TargetPrefix, 5) + r1 := startAgent(t, cfg, srcA, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + require.ErrorIs(t, r1.stop(t), context.Canceled) + + // Repointing at a different source cluster: the mirrored data on the + // target is no longer provably "this link's own" relative to the new + // source, so RequireEmpty re-arms and trips on it. + srcB := startEtcd(t, nil) + _, err := srcB.Put(t.Context(), "/src/other", "different-world") + require.NoError(t, err) + + cfg2 := cfg + cfg2.Epoch = 2 + r2 := startAgent(t, cfg2, srcB, dst) + runErr := r2.waitErr(t, 20*time.Second) + var ev *mirroragent.EmptyTargetViolationError + require.ErrorAs(t, runErr, &ev, "got: %v", runErr) + assert.Equal(t, mirroragent.ResyncReasonClusterIDMismatch, r2.agent.Snapshot().LastResyncReason) + }) + + t.Run("FreshSourceOverwriteAndPruneConverges", func(t *testing.T) { + srcA := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwrite + + want := putN(t, srcA, cfg.SourcePrefix, cfg.TargetPrefix, 5) + r1 := startAgent(t, cfg, srcA, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + require.ErrorIs(t, r1.stop(t), context.Canceled) + + srcB := startEtcd(t, nil) + _, err := srcB.Put(t.Context(), "/src/fresh", "new-world") + require.NoError(t, err) + + cfg2 := cfg + cfg2.Epoch = 2 + r2 := startAgent(t, cfg2, srcB, dst) + // The mismatch forces genesis with a mandatory mark-and-sweep: srcA's + // five keys are orphans against srcB and must be pruned even though + // the mode is plain Overwrite. + waitTargetData(t, dst, cfg, 30*time.Second, map[string]string{"/dst/fresh": "new-world"}) + snap := r2.agent.Snapshot() + assert.Equal(t, mirroragent.ResyncReasonClusterIDMismatch, snap.LastResyncReason) + f, _ := readFence(t, dst, cfg) + assert.False(t, f.PrunePending) + assert.Equal(t, snap.SourceClusterID, f.SourceClusterID, + "the checkpoint must re-bind to the new source cluster identity") + }) +} + +// breakableWatchClient wraps a source client so the test can (a) kill the +// live watch stream and (b) serve injected already-compacted watch responses +// — the deterministic stand-in for "retention outran the watch". +type breakableWatchClient struct { + mirroragent.Client + inject atomic.Bool + + mu sync.Mutex + cancels []context.CancelFunc +} + +func (c *breakableWatchClient) Watch( + ctx context.Context, key string, opts ...clientv3.OpOption, +) clientv3.WatchChan { + if c.inject.Load() { + ch := make(chan clientv3.WatchResponse, 1) + ch <- clientv3.WatchResponse{Canceled: true, CompactRevision: 3} + close(ch) + return ch + } + wctx, cancel := context.WithCancel(ctx) + c.mu.Lock() + c.cancels = append(c.cancels, cancel) + c.mu.Unlock() + return c.Client.Watch(wctx, key, opts...) +} + +func (c *breakableWatchClient) breakWatches() { + c.mu.Lock() + defer c.mu.Unlock() + for _, cancel := range c.cancels { + cancel() + } + c.cancels = nil +} + +// TestResyncLoopLatch covers the livelock detector: consecutive compaction +// failures without an intervening steady state latch ResyncLoopDetected, the +// latch does not self-clear while the loop continues, and it clears only +// when steady state is finally reached. +func TestResyncLoopLatch(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.ResyncLoopThreshold = 2 + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 8) + bsrc := &breakableWatchClient{Client: src} + r := startAgent(t, cfg, bsrc, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + waitSnap(t, r.agent, 10*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + // Kill the live watch; every re-watch now lands below the "compact + // revision" — the retention < scan-time livelock signature. + bsrc.inject.Store(true) + bsrc.breakWatches() + + snap := waitSnap(t, r.agent, 20*time.Second, "first forced resync", + func(s mirroragent.Snapshot) bool { return s.ForcedResyncCount >= 1 }) + assert.Equal(t, mirroragent.ResyncReasonCompacted, snap.LastResyncReason) + waitSnap(t, r.agent, 20*time.Second, "livelock latch", + func(s mirroragent.Snapshot) bool { return s.ResyncLoopDetected }) + + // The latch must hold while the loop continues. + time.Sleep(400 * time.Millisecond) + assert.True(t, r.agent.Snapshot().ResyncLoopDetected, "the latch must not self-clear mid-loop") + + // Heal the source; the next attempt converges. + bsrc.inject.Store(false) + _, err := src.Put(t.Context(), "/src/healed", "ok") + require.NoError(t, err) + want["/dst/healed"] = "ok" + waitTargetData(t, dst, cfg, 30*time.Second, want) + + // Steady state = a successfully applied TAIL response; only that clears + // the latch (scan convergence alone does not). A single post-heal put can + // race the recovery attempt's restart backoff and land below the scan's + // R0 — applied by the scan, leaving the live tail with nothing to + // deliver — so write a fresh key per poll tick until one arrives live. + steady := 0 + waitSnap(t, r.agent, 20*time.Second, "latch clears at steady state", + func(s mirroragent.Snapshot) bool { + if !s.ResyncLoopDetected && !s.Compacted { + return true + } + steady++ + _, perr := src.Put(t.Context(), fmt.Sprintf("/src/steady-%d", steady), "ok") + require.NoError(t, perr) + return false + }) +} + +// rangeRecordingClient records every Get's [start, end) window, to prove +// exclusion by range decomposition rather than client-side filtering. +type rangeRecordingClient struct { + mirroragent.Client + mu sync.Mutex + windows [][2]string +} + +func (c *rangeRecordingClient) Get( + ctx context.Context, key string, opts ...clientv3.OpOption, +) (*clientv3.GetResponse, error) { + op := clientv3.OpGet(key, opts...) + c.mu.Lock() + c.windows = append(c.windows, [2]string{string(op.KeyBytes()), string(op.RangeBytes())}) + c.mu.Unlock() + return c.Client.Get(ctx, key, opts...) +} + +func (c *rangeRecordingClient) recorded() [][2]string { + c.mu.Lock() + defer c.mu.Unlock() + return append([][2]string(nil), c.windows...) +} + +// TestExcludeRangeElision covers exclude handling at the RPC level: no +// source Get window may even overlap an excluded range — excluded data is +// never transferred, not fetched-then-dropped. The mode is deliberately +// OverwriteAndPrune so the assertion also covers the reconcile/prune pass +// (the code path every forced resync and drain repair runs): operators +// exclude e.g. /secrets/ precisely so those values never leave the source +// network, resyncs included. +func TestExcludeRangeElision(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwriteAndPrune + cfg.ExcludePrefixes = []string{"/src/skip/"} + cfg.PageKeyLimit = 3 // several pages, several windows + + ctx := t.Context() + want := map[string]string{} + for i := range 10 { + a := fmt.Sprintf("/src/aa-%02d", i) + z := fmt.Sprintf("/src/zz-%02d", i) + s := fmt.Sprintf("/src/skip/%02d", i) + for _, kv := range [][2]string{{a, "a"}, {z, "z"}, {s, "never"}} { + _, err := src.Put(ctx, kv[0], kv[1]) + require.NoError(t, err) + } + want["/dst/"+strings.TrimPrefix(a, "/src/")] = "a" + want["/dst/"+strings.TrimPrefix(z, "/src/")] = "z" + } + + rsrc := &rangeRecordingClient{Client: src} + r := startAgent(t, cfg, rsrc, dst) + snap := waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + waitTargetData(t, dst, cfg, 10*time.Second, want) + assert.EqualValues(t, 20, snap.InitialSyncTotalKeyCount, + "excluded keys must not appear in the InitialSync denominator") + + // A live write under the excluded prefix must not arrive either (the + // watch filters client-side; only range reads decompose). + _, err := src.Put(ctx, "/src/skip/live", "never") + require.NoError(t, err) + _, err = src.Put(ctx, "/src/aa-live", "a") + require.NoError(t, err) + want["/dst/aa-live"] = "a" + waitTargetData(t, dst, cfg, 10*time.Second, want) + + exclStart, exclEnd := "/src/skip/", "/src/skip0" + for _, w := range rsrc.recorded() { + start, end := w[0], w[1] + if end == "" { + end = start + "\x00" // point Get + } + overlaps := start < exclEnd && (end == "\x00" || end > exclStart) + assert.False(t, overlaps, + "source Get window [%q, %q) overlaps the excluded range — exclusion must be server-side elision", + w[0], w[1]) + } +} + +// TestOversizedPermanentBands covers both oversize failure bands: the +// server's request-size reject and the gRPC client send cap. Both classify +// Permanent with distinct causes and a redacted key — never throttling, +// never a retry loop. +func TestOversizedPermanentBands(t *testing.T) { + //nolint:dupl // the two bands intentionally mirror each other with different servers/causes + t.Run("ServerRejectBand", func(t *testing.T) { + src := startEtcd(t, func(c *embed.Config) { c.MaxRequestBytes = 8 << 20 }) + dst := startEtcd(t, nil) // default ~1.5MiB request ceiling + cfg := baseCfg("/src/", "/dst/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwrite + + seed := bigValueClient(t, src) + _, err := seed.Put(t.Context(), "/src/poison-key-server", strings.Repeat("v", 1600*1024)) + require.NoError(t, err) + + countingDst := &countingClient{Client: dst} + r := startAgent(t, cfg, src, countingDst) + runErr := r.waitErr(t, 30*time.Second) + var tle *mirroragent.TooLargeError + require.ErrorAs(t, runErr, &tle, "got: %v", runErr) + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + assert.Contains(t, runErr.Error(), "request is too large", + "the server reject band must surface its distinct cause") + assert.Contains(t, tle.Key, "/dst/…", "the offending key must be surfaced redacted") + assert.NotContains(t, tle.Key, "poison", "raw key bytes must never surface") + + // Permanent means no retry loop: fence claim + the poison attempt + // (single-revision set — no shrink) + slack, counted over the WHOLE + // run (Run has returned, so this bounds every commit ever made). + assert.LessOrEqual(t, countingDst.txns.Load(), int64(4), + "a permanent oversize must never be retried") + assert.Equal(t, mirroragent.PhaseFailed, r.agent.Snapshot().Phase) + }) + + //nolint:dupl // the two bands intentionally mirror each other with different servers/causes + t.Run("ClientSendCapBand", func(t *testing.T) { + src := startEtcd(t, func(c *embed.Config) { c.MaxRequestBytes = 8 << 20 }) + // The target server would accept it; the CLIENT's 2MiB send cap is + // the limit under test. + dst := startEtcd(t, func(c *embed.Config) { c.MaxRequestBytes = 8 << 20 }) + cfg := baseCfg("/src/", "/dst/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwrite + + seed := bigValueClient(t, src) + _, err := seed.Put(t.Context(), "/src/poison-key-client", strings.Repeat("v", 3<<20)) + require.NoError(t, err) + + countingDst := &countingClient{Client: dst} + r := startAgent(t, cfg, src, countingDst) + runErr := r.waitErr(t, 30*time.Second) + var tle *mirroragent.TooLargeError + require.ErrorAs(t, runErr, &tle, "got: %v", runErr) + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + assert.Contains(t, runErr.Error(), "larger than max", + "the client send cap band must surface its distinct cause") + assert.NotContains(t, tle.Key, "poison") + assert.LessOrEqual(t, countingDst.txns.Load(), int64(4), + "a permanent client-cap oversize must never be retried") + snap := r.agent.Snapshot() + assert.Equal(t, mirroragent.ClassPermanent, snap.LastErrorClass, + "the send cap shares ResourceExhausted with throttling and must NOT classify Throttle") + assert.False(t, snap.Throttled) + }) +} + +// TestCorruptCheckpointFailsClosed covers the fail-closed contract: garbage +// or an unknown-version document at the reserved key stops the agent +// permanently — no genesis, no resync, zero writes. +func TestCorruptCheckpointFailsClosed(t *testing.T) { + cases := []struct { + name string + raw string + }{ + {name: "Garbage", raw: "\x00\x01 not a checkpoint"}, + {name: "FutureVersion", raw: `{"v":99,"linkUID":"test-link","epoch":1,"role":"Mirror",` + + `"watermark":10,"sourceClusterID":"1","targetClusterID":"2"}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + + ctx := t.Context() + _, err := dst.Put(ctx, checkpointKey(cfg), tc.raw) + require.NoError(t, err) + preRev, err := dst.Get(ctx, "\x00", clientv3.WithFromKey(), clientv3.WithCountOnly()) + require.NoError(t, err) + + r := startAgent(t, cfg, src, dst) + runErr := r.waitErr(t, 20*time.Second) + var ci *mirroragent.CheckpointInvalidError + require.ErrorAs(t, runErr, &ci, "got: %v", runErr) + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + snap := r.agent.Snapshot() + assert.Equal(t, mirroragent.PhaseFailed, snap.Phase) + assert.EqualValues(t, 0, snap.ForcedResyncCount, "fail closed means NO resync") + + // Zero writes of any kind: the target revision did not move and + // the reserved key still holds the exact original bytes. + post, err := dst.Get(ctx, checkpointKey(cfg)) + require.NoError(t, err) + require.Len(t, post.Kvs, 1) + assert.Equal(t, tc.raw, string(post.Kvs[0].Value)) + assert.Equal(t, preRev.Header.Revision, post.Header.Revision, + "the agent must not have written anything to the target") + }) + } +} + +// versionStubClient overrides the maintenance Status version — the seam for +// the version-floor probe. +type versionStubClient struct { + mirroragent.Client + version string +} + +func (c *versionStubClient) Status( + ctx context.Context, endpoint string, +) (*clientv3.StatusResponse, error) { + resp, err := c.Client.Status(ctx, endpoint) + if err != nil { + return nil, err + } + resp.Version = c.version + return resp, nil +} + +// TestVersionFloor covers the declared >=3.4 hard floor: a source below it +// fails permanently at connect, before any scan read or target write. +func TestVersionFloor(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + + countingSrc := &countingClient{Client: src} + countingDst := &countingClient{Client: dst} + stubSrc := &versionStubClient{Client: countingSrc, version: "3.3.0"} + + r := startAgent(t, cfg, stubSrc, countingDst) + runErr := r.waitErr(t, 20*time.Second) + var uv *mirroragent.UnsupportedVersionError + require.ErrorAs(t, runErr, &uv, "got: %v", runErr) + assert.Equal(t, "source", uv.Side) + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + assert.Equal(t, mirroragent.PhaseFailed, r.agent.Snapshot().Phase) + + assert.EqualValues(t, 0, countingSrc.gets.Load(), "no scan read may precede the version gate") + assert.EqualValues(t, 0, countingDst.txns.Load(), "no target write may precede the version gate") +} + +// TestProgressNotifyAdvancesIdleWatermark is the RequestProgress metadata +// regression test: on an idle prefix the watermark must still advance via +// client-driven progress requests. It fails if the RequestProgress context +// metadata ever diverges from the Watch context (watcher gRPC streams are +// keyed by outgoing metadata, and the engine watches WithRequireLeader). +func TestProgressNotifyAdvancesIdleWatermark(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.ProgressInterval = 150 * time.Millisecond + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + r := startAgent(t, cfg, src, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + + // The mirrored prefix goes idle; only out-of-prefix writes move the + // cluster revision. + ctx := t.Context() + var outRev int64 + for i := range 5 { + resp, err := src.Put(ctx, fmt.Sprintf("/elsewhere/%d", i), "x") + require.NoError(t, err) + outRev = resp.Header.Revision + } + + snap := waitSnap(t, r.agent, 20*time.Second, "watermark advance on an idle prefix", + func(s mirroragent.Snapshot) bool { return s.Watermark >= outRev }) + assert.GreaterOrEqual(t, snap.SourceRevision, outRev) + f, _ := readFence(t, dst, cfg) + assert.GreaterOrEqual(t, f.Watermark, outRev, + "the fenced checkpoint — not just the snapshot — must carry the progress watermark") +} + +// duplicatingWatchClient delivers every data-bearing watch response twice. +type duplicatingWatchClient struct { + mirroragent.Client +} + +func (c *duplicatingWatchClient) Watch( + ctx context.Context, key string, opts ...clientv3.OpOption, +) clientv3.WatchChan { + in := c.Client.Watch(ctx, key, opts...) + out := make(chan clientv3.WatchResponse) + go func() { + defer close(out) + for wr := range in { + out <- wr + if len(wr.Events) > 0 && wr.Err() == nil { + out <- wr + } + } + }() + return out +} + +// TestDuplicateReplayIdempotent covers replay idempotence: duplicate event +// delivery (the reflector's overlap case, forced here for every response) +// re-puts value-identical keys — no drift, no divergence, and the drain +// verification still proves per-side equality. +func TestDuplicateReplayIdempotent(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + dupSrc := &duplicatingWatchClient{Client: src} + r := startAgent(t, cfg, dupSrc, dst) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + ctx := t.Context() + want := map[string]string{} + for i := range 10 { + k := fmt.Sprintf("dup-%02d", i) + _, err := src.Put(ctx, cfg.SourcePrefix+k, "v") + require.NoError(t, err) + want[cfg.TargetPrefix+k] = "v" + } + _, err := src.Delete(ctx, "/src/dup-03") + require.NoError(t, err) + delete(want, "/dst/dup-03") + + waitTargetData(t, dst, cfg, 20*time.Second, want) + snap := r.agent.Snapshot() + assert.EqualValues(t, 0, snap.ForcedResyncCount) + assert.Empty(t, snap.LastError, "duplicate delivery must not surface any error") + + // The strongest idempotence proof: drain verification counts both sides + // and only cuts over on exact equality. + r.agent.RequestDrain() + require.NoError(t, r.waitErr(t, 30*time.Second), "a completed drain returns nil") + final := r.agent.Snapshot() + require.NotNil(t, final.Cutover) + assert.EqualValues(t, 9, final.Cutover.SourceKeyCount) + assert.EqualValues(t, 9, final.Cutover.TargetKeyCount) + assert.EqualValues(t, 9, final.SourceKeyCount, + "the always-on per-side key counts must be populated by the verification pass") + assert.EqualValues(t, 9, final.TargetKeyCount) + assert.True(t, final.CutoverReady) +} + +// ambiguousTxnClient makes the next successful Commit report an ambiguous +// timeout: the Txn commits server-side but the caller sees DeadlineExceeded +// — the classic blackholed-NLB lost-response scenario. +type ambiguousTxnClient struct { + mirroragent.Client + arm atomic.Bool +} + +func (c *ambiguousTxnClient) Txn(ctx context.Context) clientv3.Txn { + return &ambiguousTxn{inner: c.Client.Txn(ctx), c: c} +} + +type ambiguousTxn struct { + inner clientv3.Txn + c *ambiguousTxnClient +} + +func (t *ambiguousTxn) If(cs ...clientv3.Cmp) clientv3.Txn { t.inner = t.inner.If(cs...); return t } +func (t *ambiguousTxn) Then(ops ...clientv3.Op) clientv3.Txn { + t.inner = t.inner.Then(ops...) + return t +} +func (t *ambiguousTxn) Else(ops ...clientv3.Op) clientv3.Txn { + t.inner = t.inner.Else(ops...) + return t +} +func (t *ambiguousTxn) Commit() (*clientv3.TxnResponse, error) { + resp, err := t.inner.Commit() + if err == nil && resp.Succeeded && t.c.arm.CompareAndSwap(true, false) { + return nil, context.DeadlineExceeded + } + return resp, err +} + +// TestAmbiguousCommitAdopted covers the spec's re-read-recompute-retry rule +// for fenced Txns: when a committed Txn's response is lost, the retry's +// compare fails against the agent's OWN write — the engine must recognize +// the stored fence as this attempt's value and adopt it, never misreport a +// permanent fence violation. +func TestAmbiguousCommitAdopted(t *testing.T) { + t.Run("SteadyApply", func(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + amb := &ambiguousTxnClient{Client: dst} + r := startAgent(t, cfg, src, amb) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + amb.arm.Store(true) + putResp, err := src.Put(t.Context(), "/src/ambiguous", "survives") + require.NoError(t, err) + waitTargetData(t, dst, cfg, 20*time.Second, map[string]string{"/dst/ambiguous": "survives"}) + + snap := waitSnap(t, r.agent, 20*time.Second, "recovered to Syncing", + func(s mirroragent.Snapshot) bool { + return s.Phase == mirroragent.PhaseSyncing && s.Watermark >= putResp.Header.Revision + }) + assert.EqualValues(t, 0, snap.ForcedResyncCount) + f, _ := readFence(t, dst, cfg) + assert.Equal(t, putResp.Header.Revision, f.Watermark, + "the adopted commit's checkpoint is the authoritative watermark") + select { + case runErr := <-r.done: + t.Fatalf("Run returned after an ambiguous commit: %v", runErr) + default: + } + }) + + t.Run("DrainRoleFlip", func(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + // Progress notifications on the quiesced source carry hdrRev == + // watermark and are skipped without a Txn, so the armed injection + // deterministically hits the role-flip Txn. (The interval must stay + // short — the ticker is also what wakes consume to check the drain + // gate.) + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 5) + amb := &ambiguousTxnClient{Client: dst} + r := startAgent(t, cfg, src, amb) + waitTargetData(t, dst, cfg, 20*time.Second, want) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + // The role-flip Txn commits but its response is lost: the retry must + // recognize the stored Primary fence as its own write and complete + // the drain — the pre-fix behavior wedged the cutover as a permanent + // fence violation with the target already flipped. + amb.arm.Store(true) + r.agent.RequestDrain() + require.NoError(t, r.waitErr(t, 30*time.Second), "a completed drain returns nil") + + snap := r.agent.Snapshot() + assert.Equal(t, mirroragent.PhaseDrained, snap.Phase) + assert.True(t, snap.CutoverReady) + require.NotNil(t, snap.Cutover) + assert.False(t, snap.Cutover.VerifiedTime.IsZero()) + f, _ := readFence(t, dst, cfg) + assert.Equal(t, mirroragent.RolePrimary, f.Role) + }) +} + +// claimRaceClient injects an OLD-epoch fence write between loadFence's read +// (which saw no key) and this generation's genesis claim Txn — the rolling- +// redeploy race where the outgoing generation lands its last checkpoint in +// the read/claim window. +type claimRaceClient struct { + mirroragent.Client + raw *clientv3.Client + key string + stale string + armed atomic.Bool +} + +func (c *claimRaceClient) Txn(ctx context.Context) clientv3.Txn { + if c.armed.CompareAndSwap(true, false) { + if _, err := c.raw.Put(ctx, c.key, c.stale); err != nil { + panic("claimRaceClient: injecting stale fence: " + err.Error()) + } + } + return c.Client.Txn(ctx) +} + +// TestGenesisClaimRaceTakesOver: a genesis fence claim that loses the race +// against an older generation's write must adopt the raced mod revision and +// retry the takeover — not fail the agent with a permanent FenceError. +func TestGenesisClaimRaceTakesOver(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.Epoch = 2 + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 5) + + stale, err := mirroragent.FenceValue{ + LinkUID: cfg.LinkUID, + Epoch: 1, + Role: mirroragent.RoleMirror, + Watermark: 1, + }.Encode() + require.NoError(t, err) + + race := &claimRaceClient{Client: dst, raw: dst, key: checkpointKey(cfg), stale: stale} + race.armed.Store(true) + r := startAgent(t, cfg, src, race) + + waitTargetData(t, dst, cfg, 20*time.Second, want) + snap := waitSnap(t, r.agent, 20*time.Second, "Syncing after claim-race takeover", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + assert.EqualValues(t, 0, snap.ForcedResyncCount) + f, _ := readFence(t, dst, cfg) + assert.Equal(t, int64(2), f.Epoch, "the new generation must have taken the fence over") + select { + case runErr := <-r.done: + t.Fatalf("Run returned after a claim race: %v", runErr) + default: + } +} + +// replayLoopWatchClient drives the exact shape of the resync livelock the +// detector exists for: every genesis watch delivers ONE fabricated in-scope +// event (so the replay buffer is never empty) and then dies; every tail +// re-watch reports already-compacted. Heal() restores real watches. +type replayLoopWatchClient struct { + mirroragent.Client + raw *clientv3.Client + srcPrefix string + healed atomic.Bool + + mu sync.Mutex + calls int +} + +func (c *replayLoopWatchClient) Watch( + ctx context.Context, key string, opts ...clientv3.OpOption, +) clientv3.WatchChan { + if c.healed.Load() { + return c.Client.Watch(ctx, key, opts...) + } + c.mu.Lock() + c.calls++ + n := c.calls + c.mu.Unlock() + ch := make(chan clientv3.WatchResponse, 1) + if n%2 == 0 { + // Tail re-watch: retention outran the watch. + ch <- clientv3.WatchResponse{Canceled: true, CompactRevision: 3} + close(ch) + return ch + } + // Genesis watch: one fabricated in-scope event at an existing revision, + // then channel death. The replay buffer keeps the event and genesis + // applies it — the exact sequence that must NOT reset the livelock + // detector mid-resync. + head, err := c.raw.Get(context.Background(), c.srcPrefix, clientv3.WithPrefix(), clientv3.WithCountOnly()) + if err != nil { + close(ch) + return ch + } + ch <- clientv3.WatchResponse{ + Header: *head.Header, + Events: []*clientv3.Event{{ + Type: clientv3.EventTypePut, + Kv: &mvccpb.KeyValue{ + Key: []byte(c.srcPrefix + "replay-marker"), + Value: []byte("replayed"), + ModRevision: head.Header.Revision, + }, + }}, + } + close(ch) + return ch +} + +// TestResyncLoopLatchWithReplay: the livelock detector must latch even when +// every forced resync's replay buffer applies events — a churning source is +// the CANONICAL livelock trigger (retention < scan+apply time), and replay +// applies run inside the resync being counted, so they must never count as +// steady state. +func TestResyncLoopLatchWithReplay(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.ResyncLoopThreshold = 2 + + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 8) + loop := &replayLoopWatchClient{Client: src, raw: src, srcPrefix: cfg.SourcePrefix} + r := startAgent(t, cfg, loop, dst) + + snap := waitSnap(t, r.agent, 30*time.Second, "livelock latch despite replay applies", + func(s mirroragent.Snapshot) bool { return s.ResyncLoopDetected }) + assert.GreaterOrEqual(t, snap.ForcedResyncCount, int64(2)) + + // Heal: real watches again. A write applied during a LIVE tail clears + // the latch; writes swallowed by the healing resync's own scan do not, + // so keep writing until the agent has settled into a live tail. (This + // test pins the latch behavior; byte-exactness is covered elsewhere.) + loop.healed.Store(true) + deadline := time.Now().Add(30 * time.Second) + for i := 0; r.agent.Snapshot().ResyncLoopDetected; i++ { + require.True(t, time.Now().Before(deadline), + "latch never cleared after healing despite live writes") + _, err := src.Put(t.Context(), cfg.SourcePrefix+fmt.Sprintf("steady-%d", i), "ok") + require.NoError(t, err) + time.Sleep(150 * time.Millisecond) + } +} + +// TestDrainCompletesWithoutProgressTrust: on a source below the +// progress-notify trust floor (3.4.x < 3.4.25 / 3.5.x < 3.5.8) the drain +// target must be derived from the highest in-scope mod revision, not the +// cluster revision — out-of-prefix writes would otherwise park the drain in +// PhaseSyncing forever with no error (the watermark cannot advance past the +// last in-prefix event without trusted progress notifications). +func TestDrainCompletesWithoutProgressTrust(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 6) + stubSrc := &versionStubClient{Client: src, version: "3.5.7"} + r := startAgent(t, cfg, stubSrc, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + // Out-of-prefix writes push the cluster revision past every in-prefix + // event — the quiesced-prefix drain must still terminate. + ctx := t.Context() + var clusterRev int64 + for i := range 5 { + resp, err := src.Put(ctx, fmt.Sprintf("/elsewhere/%d", i), "x") + require.NoError(t, err) + clusterRev = resp.Header.Revision + } + + r.agent.RequestDrain() + require.NoError(t, r.waitErr(t, 30*time.Second), "the drain must terminate below the trust floor") + snap := r.agent.Snapshot() + assert.Equal(t, mirroragent.PhaseDrained, snap.Phase) + assert.True(t, snap.CutoverReady) + require.NotNil(t, snap.Cutover) + assert.Less(t, snap.Cutover.DrainTargetRevision, clusterRev, + "the drain target must be the in-scope high-water mark, not the cluster revision") + f, _ := readFence(t, dst, cfg) + assert.Equal(t, mirroragent.RolePrimary, f.Role) +} + +// TestProgressNotifyNotTrustedBelowFloor: on a 3.5.7 source the watermark +// must NOT advance on an idle prefix (progress notifications are unreliable +// below 3.4.25/3.5.8 and may report revisions ahead of delivered events — +// trusting them checkpoints past undelivered data), while applies still +// advance it. +func TestProgressNotifyNotTrustedBelowFloor(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.ProgressInterval = 100 * time.Millisecond + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + stubSrc := &versionStubClient{Client: src, version: "3.5.7"} + r := startAgent(t, cfg, stubSrc, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + base := waitSnap(t, r.agent, 10*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + ctx := t.Context() + var outRev int64 + for i := range 5 { + resp, err := src.Put(ctx, fmt.Sprintf("/elsewhere/%d", i), "x") + require.NoError(t, err) + outRev = resp.Header.Revision + } + // Several progress intervals: an untrusted notification must not move + // the watermark past the last applied in-prefix revision. + time.Sleep(6 * cfg.ProgressInterval) + mid := r.agent.Snapshot() + assert.Less(t, mid.Watermark, outRev, + "the watermark must not advance from untrusted progress notifications") + assert.Equal(t, base.Watermark, mid.Watermark) + + // An applied in-prefix event still advances it. + putResp, err := src.Put(ctx, "/src/applied", "yes") + require.NoError(t, err) + want["/dst/applied"] = "yes" + waitTargetData(t, dst, cfg, 20*time.Second, want) + waitSnap(t, r.agent, 10*time.Second, "watermark advances on applies", + func(s mirroragent.Snapshot) bool { return s.Watermark >= putResp.Header.Revision }) +} + +// flakyEndpointClient reports two endpoints, the first of which is +// blackholed for the maintenance Status probe. +type flakyEndpointClient struct { + mirroragent.Client + bad string +} + +func (c *flakyEndpointClient) Endpoints() []string { + return append([]string{c.bad}, c.Client.Endpoints()...) +} + +func (c *flakyEndpointClient) Status( + ctx context.Context, endpoint string, +) (*clientv3.StatusResponse, error) { + if endpoint == c.bad { + return nil, status.Error(codes.Unavailable, "blackholed endpoint") + } + return c.Client.Status(ctx, endpoint) +} + +// TestProbeRotatesEndpoints: Status dials the named endpoint directly +// (bypassing the balancer), so the connect probe must rotate through the +// endpoint list — one dead member/NAT mapping must not wedge the agent in +// Connecting while healthy endpoints exist. +func TestProbeRotatesEndpoints(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + flaky := &flakyEndpointClient{Client: src, bad: "http://127.0.0.1:1"} + r := startAgent(t, cfg, flaky, dst) + waitSnap(t, r.agent, 20*time.Second, "Syncing despite a blackholed first endpoint", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + waitTargetData(t, dst, cfg, 10*time.Second, want) +} + +// TestPruneRefusesForeignFence: a prune pass that encounters ANOTHER link's +// reserved fence key inside this link's destination prefix (overlapping +// destination prefixes — e.g. a second EtcdMirror at a nested prefix) must +// stop loudly with a PrefixConflictError instead of deleting the sibling's +// fence and data as orphans. +func TestPruneRefusesForeignFence(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwriteAndPrune + + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + + // A sibling mirror lives at the nested prefix /dst/sub/ with its own + // fence and data. + foreignFence, err := mirroragent.FenceValue{ + LinkUID: "other-link", + Epoch: 1, + Role: mirroragent.RoleMirror, + Watermark: 7, + }.Encode() + require.NoError(t, err) + ctx := t.Context() + foreignKey := "/dst/sub/" + mirroragent.DefaultCheckpointKeySuffix + _, err = dst.Put(ctx, foreignKey, foreignFence) + require.NoError(t, err) + _, err = dst.Put(ctx, "/dst/sub/data", "sibling-owned") + require.NoError(t, err) + + r := startAgent(t, cfg, src, dst) + runErr := r.waitErr(t, 30*time.Second) + var pc *mirroragent.PrefixConflictError + require.ErrorAs(t, runErr, &pc, "got: %v", runErr) + assert.Equal(t, "other-link", pc.OwnerLinkUID) + assert.NotContains(t, pc.Key, "sub", "the foreign key must be surfaced redacted") + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + assert.Equal(t, mirroragent.PhaseFailed, r.agent.Snapshot().Phase) + + // Nothing of the sibling's was destroyed. + got, err := dst.Get(ctx, foreignKey) + require.NoError(t, err) + require.Len(t, got.Kvs, 1, "the sibling's fence key must survive") + assert.Equal(t, foreignFence, string(got.Kvs[0].Value)) + data, err := dst.Get(ctx, "/dst/sub/data") + require.NoError(t, err) + require.Len(t, data.Kvs, 1, "the sibling's data must survive") +} + +// TestShrinkOnTargetTxnLimit: the target is a foreign cluster whose +// --max-txn-ops the operator cannot inspect. A multi-revision flush set the +// target rejects must be re-committed at revision granularity (one shrink +// attempt) instead of failing the agent permanently — only an irreducible +// single revision is a true permanent oversize. +func TestShrinkOnTargetTxnLimit(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, func(c *embed.Config) { c.MaxTxnOps = 8 }) + cfg := baseCfg("/src/", "/dst/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwrite + cfg.MaxTxnOps = 64 // engine batches far past the target's limit of 8 + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 20) + + r := startAgent(t, cfg, src, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + snap := waitSnap(t, r.agent, 20*time.Second, "Syncing after shrink", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + assert.EqualValues(t, 0, snap.ForcedResyncCount) + assert.Empty(t, snap.LastError, "a successful shrink must clear the recorded error") + + // Live tail keeps working under the same limit. + _, err := src.Put(t.Context(), "/src/after", "ok") + require.NoError(t, err) + want["/dst/after"] = "ok" + waitTargetData(t, dst, cfg, 10*time.Second, want) +} + +// txnFailingClient fails every Txn with a transient error while armed. +type txnFailingClient struct { + mirroragent.Client + failing atomic.Bool +} + +func (c *txnFailingClient) Txn(ctx context.Context) clientv3.Txn { + return &failableTxn{inner: c.Client.Txn(ctx), c: c} +} + +type failableTxn struct { + inner clientv3.Txn + c *txnFailingClient +} + +func (t *failableTxn) If(cs ...clientv3.Cmp) clientv3.Txn { t.inner = t.inner.If(cs...); return t } +func (t *failableTxn) Then(ops ...clientv3.Op) clientv3.Txn { t.inner = t.inner.Then(ops...); return t } +func (t *failableTxn) Else(ops ...clientv3.Op) clientv3.Txn { t.inner = t.inner.Else(ops...); return t } +func (t *failableTxn) Commit() (*clientv3.TxnResponse, error) { + if t.c.failing.Load() { + return nil, status.Error(codes.Unavailable, "injected target stall") + } + return t.inner.Commit() +} + +// watchCountingClient counts Watch calls. +type watchCountingClient struct { + mirroragent.Client + watches atomic.Int64 +} + +func (c *watchCountingClient) Watch( + ctx context.Context, key string, opts ...clientv3.OpOption, +) clientv3.WatchChan { + c.watches.Add(1) + return c.Client.Watch(ctx, key, opts...) +} + +// TestSustainedTargetBackoffCancelsWatch: while the target stalls, clientv3 +// buffers undelivered source watch responses without bound — after a few +// backoff rounds the engine must cancel the source watch (bounding memory) +// and, once the target heals, resume from the checkpoint watermark on a +// fresh watch with nothing lost. +func TestSustainedTargetBackoffCancelsWatch(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + countingSrc := &watchCountingClient{Client: src} + failingDst := &txnFailingClient{Client: dst} + r := startAgent(t, cfg, countingSrc, failingDst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + waitSnap(t, r.agent, 10*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + before := countingSrc.watches.Load() + + // Stall the target and keep the source churning into the stalled apply. + failingDst.failing.Store(true) + ctx := t.Context() + _, err := src.Put(ctx, "/src/stall-trigger", "x") + require.NoError(t, err) + want["/dst/stall-trigger"] = "x" + waitSnap(t, r.agent, 20*time.Second, "Degraded during the stall", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseDegraded }) + // Ride out several backoff rounds (50ms initial, 500ms cap) so the + // watch-cancel threshold (3 rounds) is comfortably crossed. + time.Sleep(1 * time.Second) + + failingDst.failing.Store(false) + for i := range 3 { + k := fmt.Sprintf("post-stall-%d", i) + _, perr := src.Put(ctx, cfg.SourcePrefix+k, "y") + require.NoError(t, perr) + want[cfg.TargetPrefix+k] = "y" + } + waitTargetData(t, dst, cfg, 30*time.Second, want) + waitSnap(t, r.agent, 20*time.Second, "Syncing after the stall", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + assert.Greater(t, countingSrc.watches.Load(), before, + "the stall must have cancelled the source watch and re-watched from the checkpoint") + assert.EqualValues(t, 0, r.agent.Snapshot().ForcedResyncCount) +} diff --git a/pkg/mirroragent/integration_reconcile_test.go b/pkg/mirroragent/integration_reconcile_test.go new file mode 100644 index 00000000..01b18719 --- /dev/null +++ b/pkg/mirroragent/integration_reconcile_test.go @@ -0,0 +1,285 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent_test + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd-operator/pkg/mirroragent" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// TestPeriodicReconcileRepairsDrift: target damage invisible to the source +// watch — a divergent value and a deleted mirrored key — is found and +// repaired byte-exactly by the periodic pass, and a later pass records +// matching per-side key counts. +func TestPeriodicReconcileRepairsDrift(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.ReconcileInterval = 500 * time.Millisecond + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 20) + + r := startAgent(t, cfg, src, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + // Land in the quiet window right after a pass completes, so the next + // pass observes both damages together. + waitSnap(t, r.agent, 20*time.Second, "first periodic pass", + func(s mirroragent.Snapshot) bool { return !s.LastReconcileTime.IsZero() }) + + ctx := t.Context() + // One commit revision for both damages: drift is replaced wholesale each + // pass, so damages split across a pass boundary would never appear in + // the same drift and the conjunction below would time out. + _, err := dst.Txn(ctx).Then( + clientv3.OpPut("/dst/key-0000", "tampered"), + clientv3.OpDelete("/dst/key-0001"), + ).Commit() + require.NoError(t, err) + + snap := waitSnap(t, r.agent, 20*time.Second, "drift observed by the periodic pass", + func(s mirroragent.Snapshot) bool { + return s.LastReconcileDrift != nil && + s.LastReconcileDrift.DivergentKeys >= 1 && + s.LastReconcileDrift.MissingKeys >= 1 + }) + assert.True(t, snap.LastReconcileDrift.Repaired, "the periodic pass must repair, not just report") + + waitTargetData(t, dst, cfg, 10*time.Second, want) + waitSnap(t, r.agent, 20*time.Second, "matching counts after repair", + func(s mirroragent.Snapshot) bool { + return s.SourceKeyCount == 20 && s.TargetKeyCount == 20 + }) +} + +// startConvergedWithOrphan converges a small mirror with the periodic pass +// enabled at the given orphan policy, then plants an orphan key directly +// under the destination prefix (no source counterpart). +func startConvergedWithOrphan(t *testing.T, deleteOrphans bool) (*agentRun, *clientv3.Client) { + t.Helper() + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.ReconcileInterval = 300 * time.Millisecond + cfg.ReconcileDeleteOrphans = deleteOrphans + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 5) + r := startAgent(t, cfg, src, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + + _, err := dst.Put(t.Context(), orphanKey, "planted") + require.NoError(t, err) + return r, dst +} + +const orphanKey = "/dst/zz-orphan" + +// TestPeriodicReconcileOrphanPolicy: ReconcileDeleteOrphans gates ONLY the +// deletion of target keys with no source counterpart — a report-only pass +// still repairs and keeps reporting the orphan pass after pass. +func TestPeriodicReconcileOrphanPolicy(t *testing.T) { + t.Run("report only", func(t *testing.T) { + r, dst := startConvergedWithOrphan(t, false) + first := waitSnap(t, r.agent, 20*time.Second, "orphan reported", + func(s mirroragent.Snapshot) bool { + return s.LastReconcileDrift != nil && s.LastReconcileDrift.OrphanKeys >= 1 + }) + // A second full pass: still reported, still not deleted. + waitSnap(t, r.agent, 20*time.Second, "a later pass reporting the orphan", + func(s mirroragent.Snapshot) bool { + return s.LastReconcileTime.After(first.LastReconcileTime) && + s.LastReconcileDrift != nil && s.LastReconcileDrift.OrphanKeys >= 1 + }) + got, err := dst.Get(t.Context(), orphanKey) + require.NoError(t, err) + require.Len(t, got.Kvs, 1, "deleteOrphans=false must never delete the orphan") + }) + + t.Run("delete", func(t *testing.T) { + r, dst := startConvergedWithOrphan(t, true) + snap := waitSnap(t, r.agent, 20*time.Second, "orphan reported and deleted", + func(s mirroragent.Snapshot) bool { + return s.LastReconcileDrift != nil && s.LastReconcileDrift.OrphanKeys >= 1 + }) + assert.True(t, snap.LastReconcileDrift.Repaired) + got, err := dst.Get(t.Context(), orphanKey) + require.NoError(t, err) + assert.Empty(t, got.Kvs, "deleteOrphans=true must delete the orphan") + }) +} + +// TestPeriodicReconcileAfterGenesisOnly: with a RequireEmpty cold start +// (no mandatory sweep) the ONLY producer of LastReconcileTime is the +// periodic pass, and even an interval much shorter than the throttled +// genesis scan never fires during it — the first pass completes after +// InitialSyncCompletionTime. +func TestPeriodicReconcileAfterGenesisOnly(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.ReconcileInterval = 50 * time.Millisecond + cfg.MaxOpsPerSecond = 100 // stretch the 300-key genesis scan to ~3s + + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 300) + + r := startAgent(t, cfg, src, dst) + snap := waitSnap(t, r.agent, 60*time.Second, "first periodic pass", + func(s mirroragent.Snapshot) bool { return !s.LastReconcileTime.IsZero() }) + require.False(t, snap.InitialSyncCompletionTime.IsZero()) + assert.False(t, snap.LastReconcileTime.Before(snap.InitialSyncCompletionTime), + "no periodic pass may run before the genesis scan completed") + assert.EqualValues(t, 300, snap.SourceKeyCount) + assert.EqualValues(t, 300, snap.TargetKeyCount) +} + +// TestPeriodicReconcilePacedRepairRecovers: a periodic pass repairing more +// than a few seconds' worth of MaxOpsPerSecond pacing cancels the live +// source watch mid-pass (the bound on clientv3's unbounded per-watcher +// buffer while the pass leaves the watch unread); the tail re-watches from +// the checkpoint watermark, so the repair still converges and live +// replication resumes afterwards. +func TestPeriodicReconcilePacedRepairRecovers(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.ReconcileInterval = 300 * time.Millisecond + cfg.MaxOpsPerSecond = 30 // 100 repairs > 3s of pacing: mid-pass cancel + + const n = 100 + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, n) + r := startAgent(t, cfg, src, dst) + waitTargetData(t, dst, cfg, 30*time.Second, want) + + // One commit revision so a single pass observes all n divergent keys and + // crosses the paced-repair cancel threshold within that pass. + ctx := t.Context() + ops := make([]clientv3.Op, 0, n) + for i := range n { + ops = append(ops, clientv3.OpPut(fmt.Sprintf("/dst/key-%04d", i), "tampered")) + } + _, err := dst.Txn(ctx).Then(ops...).Commit() + require.NoError(t, err) + + waitTargetData(t, dst, cfg, 30*time.Second, want) + // The watch the pass cancelled was re-established from the watermark: a + // fresh source write still replicates. + _, err = src.Put(ctx, cfg.SourcePrefix+"after-repair", "live") + require.NoError(t, err) + want[cfg.TargetPrefix+"after-repair"] = "live" + waitTargetData(t, dst, cfg, 20*time.Second, want) + assert.NotEqual(t, mirroragent.PhaseFailed, r.agent.Snapshot().Phase) +} + +// TestPeriodicReconcileFenceAbort: a fence taken over by a newer agent +// generation aborts the periodic pass's first repair write permanently — +// the pass rides the same fenced Txn path as every other write, so the +// stale generation cannot "repair" anything after losing the fence. +func TestPeriodicReconcileFenceAbort(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.ReconcileInterval = 300 * time.Millisecond + // Keep the progress driver quiet so the periodic repair is the only + // active write path on the idle source. + cfg.ProgressInterval = 30 * time.Second + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 5) + r := startAgent(t, cfg, src, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + + // A newer generation of this link takes the fence over externally... + cur, _ := readFence(t, dst, cfg) + takeover := cur + takeover.Epoch = cur.Epoch + 1 + val, err := takeover.Encode() + require.NoError(t, err) + ctx := t.Context() + _, err = dst.Put(ctx, checkpointKey(cfg), val) + require.NoError(t, err) + // ...then a divergent target key forces the next pass to attempt a + // fenced repair write. + _, err = dst.Put(ctx, "/dst/key-0000", "tampered") + require.NoError(t, err) + + runErr := r.waitErr(t, 30*time.Second) + var fe *mirroragent.FenceError + require.ErrorAs(t, runErr, &fe, "got: %v", runErr) + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + assert.Equal(t, mirroragent.PhaseFailed, r.agent.Snapshot().Phase) + + // The aborted pass wrote nothing after the fence moved. + got, err := dst.Get(ctx, "/dst/key-0000") + require.NoError(t, err) + require.Len(t, got.Kvs, 1) + assert.Equal(t, "tampered", string(got.Kvs[0].Value)) +} + +// TestMandatoryPassesPopulateCountsWithoutPeriodic: the always-on contract — +// with the periodic pass disabled (ReconcileInterval 0, the engine image of +// spec.reconciliation absent), the mandatory passes still populate the +// per-side key counts and stamp the pass-completion time. +func TestMandatoryPassesPopulateCountsWithoutPeriodic(t *testing.T) { + t.Run("overwrite and prune genesis", func(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwriteAndPrune + + // Pre-populated target: a stale value at a mirrored key. + _, err := dst.Put(t.Context(), "/dst/key-0000", "stale") + require.NoError(t, err) + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 10) + + r := startAgent(t, cfg, src, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + snap := waitSnap(t, r.agent, 20*time.Second, "mandatory sweep counts", + func(s mirroragent.Snapshot) bool { return !s.LastReconcileTime.IsZero() }) + assert.EqualValues(t, 10, snap.SourceKeyCount) + assert.EqualValues(t, 10, snap.TargetKeyCount) + assert.NotNil(t, snap.LastReconcileDrift, "the mandatory sweep is a full diff") + }) + + t.Run("drain verification", func(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 8) + r := startAgent(t, cfg, src, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + + r.agent.RequestDrain() + require.NoError(t, r.waitErr(t, 30*time.Second), "a completed drain returns nil") + snap := r.agent.Snapshot() + assert.Equal(t, mirroragent.PhaseDrained, snap.Phase) + require.NotNil(t, snap.Cutover) + assert.EqualValues(t, 8, snap.Cutover.SourceKeyCount) + assert.EqualValues(t, 8, snap.Cutover.TargetKeyCount) + assert.EqualValues(t, 8, snap.SourceKeyCount) + assert.EqualValues(t, 8, snap.TargetKeyCount) + assert.False(t, snap.LastReconcileTime.IsZero(), + "the count-only drain verification must stamp the pass time") + assert.Nil(t, snap.LastReconcileDrift, + "a count-only verification must not fabricate a drift") + }) +} diff --git a/pkg/mirroragent/integration_test.go b/pkg/mirroragent/integration_test.go new file mode 100644 index 00000000..dd491258 --- /dev/null +++ b/pkg/mirroragent/integration_test.go @@ -0,0 +1,531 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Integration tests for the mirror engine against two in-process embedded +// etcd servers (source + target). Each test exercises one contract from the +// Design-3 spec; intervals are shrunk via engine config knobs to keep the +// suite fast. +package mirroragent_test + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd-operator/pkg/mirroragent" + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" +) + +// TestColdStart covers scenario 1: a populated source prefix is fully +// scanned onto the target, the checkpoint/fence key carries the scan-base +// watermark, and the reserved key is excluded from the data invariants. +func TestColdStart(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 50) + r0 := sourceRevision(t, src, cfg.SourcePrefix) + + r := startAgent(t, cfg, src, dst) + snap := waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + assert.EqualValues(t, 50, snap.InitialSyncKeyCount) + assert.EqualValues(t, 50, snap.InitialSyncTotalKeyCount) + assert.EqualValues(t, 0, snap.ForcedResyncCount) + assert.Equal(t, r0, snap.Watermark) + + // Data invariant: exactly the 50 mirrored keys, reserved key excluded. + waitTargetData(t, dst, cfg, 10*time.Second, want) + raw, err := dst.Get(t.Context(), cfg.TargetPrefix, clientv3.WithPrefix(), clientv3.WithCountOnly()) + require.NoError(t, err) + assert.EqualValues(t, 51, raw.Count, + "raw range must hold the 50 data keys plus the reserved checkpoint key") + + f, _ := readFence(t, dst, cfg) + assert.Equal(t, cfg.LinkUID, f.LinkUID) + assert.Equal(t, cfg.Epoch, f.Epoch) + assert.Equal(t, mirroragent.RoleMirror, f.Role) + assert.Equal(t, r0, f.Watermark, "checkpoint watermark must be the scan base revision") + assert.False(t, f.Scanning) +} + +// TestLiveTail covers scenario 2: puts, deletes, and a multi-key source Txn +// arriving during the tail land exactly, and one source revision lands as +// ONE target Txn — every key of the source Txn plus the checkpoint write +// shares a single target mod revision (revision-aligned batching; a partial +// revision is unobservable at any point because the Txn is atomic). +func TestLiveTail(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + r := startAgent(t, cfg, src, dst) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + ctx := t.Context() + _, err := src.Put(ctx, "/src/k1", "v1") + require.NoError(t, err) + waitTargetData(t, dst, cfg, 10*time.Second, map[string]string{"/dst/k1": "v1"}) + + _, err = src.Delete(ctx, "/src/k1") + require.NoError(t, err) + waitTargetData(t, dst, cfg, 10*time.Second, map[string]string{}) + + // One multi-key source Txn = one source revision. + txnResp, err := src.Txn(ctx).Then( + clientv3.OpPut("/src/t1", "a"), + clientv3.OpPut("/src/t2", "b"), + clientv3.OpPut("/src/t3", "c"), + ).Commit() + require.NoError(t, err) + srcTxnRev := txnResp.Header.Revision + + waitTargetData(t, dst, cfg, 10*time.Second, + map[string]string{"/dst/t1": "a", "/dst/t2": "b", "/dst/t3": "c"}) + + resp, err := dst.Get(ctx, "/dst/t1", clientv3.WithRange("/dst/t4")) + require.NoError(t, err) + require.Len(t, resp.Kvs, 3) + applyRev := resp.Kvs[0].ModRevision + for _, kv := range resp.Kvs { + assert.Equal(t, applyRev, kv.ModRevision, + "all keys of one source revision must land in one target Txn") + } + f, fenceModRev := readFence(t, dst, cfg) + assert.Equal(t, applyRev, fenceModRev, + "the checkpoint must be written in the SAME Txn as the applied batch") + assert.Equal(t, srcTxnRev, f.Watermark, + "checkpoint watermark must be the applied source revision") +} + +// TestMidScanCompaction covers scenario 3, the Design-3 headline: the source +// is compacted aggressively while a rate-limited scan is in flight, and the +// scan still converges with NO forced resync — the unpinned scan plus +// watch-replay-from-R0 eliminates mid-scan compaction as a failure class. +func TestMidScanCompaction(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.MaxOpsPerSecond = 150 // ~2s scan for 300 keys + cfg.PageKeyLimit = 50 + cfg.MaxTxnOps = 20 + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 300) + r0 := sourceRevision(t, src, cfg.SourcePrefix) + + r := startAgent(t, cfg, src, dst) + waitSnap(t, r.agent, 20*time.Second, "scan started", + func(s mirroragent.Snapshot) bool { return s.InitialSyncKeyCount > 0 }) + + // Hammer the source: live writes advance the revision and compaction + // chases the head while the scan is still running. + ctx := t.Context() + for i := range 30 { + k := fmt.Sprintf("live-%03d", i) + _, err := src.Put(ctx, cfg.SourcePrefix+k, "live") + require.NoError(t, err) + want[cfg.TargetPrefix+k] = "live" + cur, err := src.Get(ctx, cfg.SourcePrefix, clientv3.WithPrefix(), clientv3.WithCountOnly()) + require.NoError(t, err) + _, _ = src.Compact(ctx, cur.Header.Revision) // errors ("already compacted") are fine + time.Sleep(50 * time.Millisecond) + } + + // Prove the compactions bit: a revision-pinned read at the scan base — + // what a mirror.Syncer-style pinned scan would issue — is now impossible. + _, err := src.Get(ctx, cfg.SourcePrefix, clientv3.WithPrefix(), clientv3.WithRev(r0)) + require.ErrorIs(t, rpctypes.Error(err), rpctypes.ErrCompacted, + "the scan base must actually be compacted for this test to mean anything") + + waitTargetData(t, dst, cfg, 30*time.Second, want) + snap := waitSnap(t, r.agent, 20*time.Second, "Syncing after compacted scan", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + assert.EqualValues(t, 0, snap.ForcedResyncCount, + "mid-scan compaction must not force a resync (class elimination)") + assert.False(t, snap.Compacted) + assert.False(t, snap.ResyncLoopDetected) +} + +// TestRestartResume covers scenario 4: a stopped engine restarted with the +// same linkUID/epoch resumes from the checkpoint in the target — zero source +// range reads (no rescan), only the watch replays the missed writes. +func TestRestartResume(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 30) + + r1 := startAgent(t, cfg, src, dst) + waitSnap(t, r1.agent, 20*time.Second, "first run Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + waitTargetData(t, dst, cfg, 10*time.Second, want) + err := r1.stop(t) + require.ErrorIs(t, err, context.Canceled) + + // Writes while the engine is down. + ctx := t.Context() + for i := range 5 { + k := fmt.Sprintf("extra-%d", i) + _, perr := src.Put(ctx, cfg.SourcePrefix+k, "late") + require.NoError(t, perr) + want[cfg.TargetPrefix+k] = "late" + } + lastRev := sourceRevision(t, src, cfg.SourcePrefix) + stopped, _ := readFence(t, dst, cfg) + + countingSrc := &countingClient{Client: src} + recordingSrc := &watchRevRecordingClient{Client: countingSrc} + r2 := startAgent(t, cfg, recordingSrc, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + snap := waitSnap(t, r2.agent, 10*time.Second, "resumed watermark", + func(s mirroragent.Snapshot) bool { return s.Watermark >= lastRev }) + + assert.EqualValues(t, 0, countingSrc.gets.Load(), + "resume must not issue ANY source range read — no rescan") + // Pin the resume revision itself: zero Gets only rules out RANGE + // rescans; a watch opened at rev 1 would replay the whole history + // (idempotently, so no data assertion can catch it) and re-transfer the + // full prefix on every restart. + revs := recordingSrc.recorded() + require.NotEmpty(t, revs) + assert.Equal(t, stopped.Watermark+1, revs[0], + "the resumed watch must open at exactly checkpointWatermark+1") + assert.EqualValues(t, 0, snap.InitialSyncKeyCount, "resume must not re-run the initial scan") + assert.EqualValues(t, 0, snap.ForcedResyncCount) + f, _ := readFence(t, dst, cfg) + assert.GreaterOrEqual(t, f.Watermark, lastRev) +} + +// TestFenceOverlap covers scenario 5: two engines share the reserved key; a +// newer epoch takes the fence over, the stale writer's next Txn fails its +// mod-revision compare and stops with the fencing error, and the target +// state is the new writer's alone. +func TestFenceOverlap(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfgA := baseCfg("/src/", "/dst/") + + putN(t, src, cfgA.SourcePrefix, cfgA.TargetPrefix, 5) + + rA := startAgent(t, cfgA, src, dst) + waitSnap(t, rA.agent, 20*time.Second, "A Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + cfgB := cfgA + cfgB.Epoch = 2 + rB := startAgent(t, cfgB, src, dst) + waitSnap(t, rB.agent, 20*time.Second, "B Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + // Both engines watch this write; only epoch 2 may land it. + _, err := src.Put(t.Context(), "/src/fence-probe", "who-wins") + require.NoError(t, err) + + errA := rA.waitErr(t, 20*time.Second) + var fe *mirroragent.FenceError + require.ErrorAs(t, errA, &fe, "stale engine must stop with the fencing error, got: %v", errA) + assert.Contains(t, fe.Detail, "epoch 2") + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(errA)) + assert.Equal(t, mirroragent.PhaseFailed, rA.agent.Snapshot().Phase) + + resp, err := dst.Get(t.Context(), "/dst/fence-probe") + require.NoError(t, err) + require.Len(t, resp.Kvs, 1, "the new writer must have applied the probe write") + assert.Equal(t, "who-wins", string(resp.Kvs[0].Value)) + f, _ := readFence(t, dst, cfgB) + assert.Equal(t, int64(2), f.Epoch, "target fence must be the new writer's alone") + assert.Equal(t, mirroragent.PhaseSyncing, rB.agent.Snapshot().Phase) +} + +// TestRoleFlipCutoverFence covers scenario 6: flipping the fence role to +// Primary (simulated cutover) makes the running engine's next apply fail +// loudly with the cutover-fence error class, and nothing lands after it. +func TestRoleFlipCutoverFence(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + r := startAgent(t, cfg, src, dst) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + // Simulated cutover: rewrite the fence with role=Primary. + f, _ := readFence(t, dst, cfg) + f.Role = mirroragent.RolePrimary + val, err := f.Encode() + require.NoError(t, err) + _, err = dst.Put(t.Context(), checkpointKey(cfg), val) + require.NoError(t, err) + + _, err = src.Put(t.Context(), "/src/after-cutover", "straggler") + require.NoError(t, err) + + runErr := r.waitErr(t, 20*time.Second) + var fe *mirroragent.FenceError + require.ErrorAs(t, runErr, &fe, "engine must stop with the fencing error, got: %v", runErr) + assert.Contains(t, fe.Detail, "Primary") + assert.Contains(t, fe.Detail, "cutover") + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + + resp, err := dst.Get(t.Context(), "/dst/after-cutover") + require.NoError(t, err) + assert.Empty(t, resp.Kvs, "no straggler apply may land after the role flip") +} + +// TestOversizedRevision covers scenario 7: a single source revision whose +// total bytes and op count both exceed the flush watermarks is applied as +// ONE oversized Txn with the checkpoint riding in it (held until it lands). +// The target's --max-txn-ops (embed MaxTxnOps) is bumped to make room. +func TestOversizedRevision(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, func(c *embed.Config) { c.MaxTxnOps = 64 }) + cfg := baseCfg("/src/", "/dst/") + cfg.MaxTxnOps = 6 // 5 data op slots + checkpoint slot + cfg.TxnFlushBytes = 2048 // the txn below is ~4.8KiB + cfg.MaxOpsPerSecond = 0 + + r := startAgent(t, cfg, src, dst) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + big := strings.Repeat("v", 600) + ops := make([]clientv3.Op, 0, 8) + want := make(map[string]string, 8) + for i := range 8 { + k := fmt.Sprintf("big-%d", i) + ops = append(ops, clientv3.OpPut("/src/"+k, big)) + want["/dst/"+k] = big + } + txnResp, err := src.Txn(t.Context()).Then(ops...).Commit() + require.NoError(t, err) + + waitTargetData(t, dst, cfg, 20*time.Second, want) + resp, err := dst.Get(t.Context(), "/dst/big-", clientv3.WithPrefix()) + require.NoError(t, err) + require.Len(t, resp.Kvs, 8) + applyRev := resp.Kvs[0].ModRevision + for _, kv := range resp.Kvs { + assert.Equal(t, applyRev, kv.ModRevision, + "an oversized source revision must be applied as ONE Txn, never split") + } + f, fenceModRev := readFence(t, dst, cfg) + assert.Equal(t, applyRev, fenceModRev, + "the checkpoint must be held and land in the same oversized Txn") + assert.Equal(t, txnResp.Header.Revision, f.Watermark) +} + +// TestInitialSyncModes covers scenario 8: the three initialSync modes plus +// excludePrefixes. Sub-scenarios share one server pair on disjoint prefixes. +func TestInitialSyncModes(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + ctx := t.Context() + + t.Run("RequireEmptyViolation", func(t *testing.T) { + cfg := baseCfg("/m1/", "/d1/") + _, err := dst.Put(ctx, "/d1/existing", "dirty") + require.NoError(t, err) + + r := startAgent(t, cfg, src, dst) + runErr := r.waitErr(t, 20*time.Second) + var ev *mirroragent.EmptyTargetViolationError + require.ErrorAs(t, runErr, &ev, "got: %v", runErr) + assert.EqualValues(t, 1, ev.KeyCount) + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + assert.Equal(t, mirroragent.PhaseFailed, r.agent.Snapshot().Phase) + // A RequireEmpty violation must write nothing, not even the fence. + resp, err := dst.Get(ctx, checkpointKey(cfg)) + require.NoError(t, err) + assert.Empty(t, resp.Kvs) + }) + + t.Run("OverwriteKeepsOrphans", func(t *testing.T) { + cfg := baseCfg("/m2/", "/d2/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwrite + _, err := dst.Put(ctx, "/d2/stale", "old") + require.NoError(t, err) + _, err = dst.Put(ctx, "/d2/orphan", "keep-me") + require.NoError(t, err) + _, err = src.Put(ctx, "/m2/stale", "new") + require.NoError(t, err) + _, err = src.Put(ctx, "/m2/fresh", "1") + require.NoError(t, err) + + r := startAgent(t, cfg, src, dst) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + waitTargetData(t, dst, cfg, 10*time.Second, map[string]string{ + "/d2/stale": "new", // overwritten with source truth + "/d2/fresh": "1", // mirrored + "/d2/orphan": "keep-me", // Overwrite leaves orphans alone + }) + }) + + t.Run("OverwriteAndPruneRemovesOrphans", func(t *testing.T) { + cfg := baseCfg("/m3/", "/d3/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwriteAndPrune + // Simulates failback onto a stale prefix: "zombie" stands for a key + // deleted on the (new-primary) source after cutover — the prune pass + // must remove it from the old copy. + _, err := dst.Put(ctx, "/d3/stale", "old") + require.NoError(t, err) + _, err = dst.Put(ctx, "/d3/zombie", "deleted-post-cutover") + require.NoError(t, err) + _, err = src.Put(ctx, "/m3/stale", "new") + require.NoError(t, err) + _, err = src.Put(ctx, "/m3/fresh", "1") + require.NoError(t, err) + + r := startAgent(t, cfg, src, dst) + snap := waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + waitTargetData(t, dst, cfg, 10*time.Second, map[string]string{ + "/d3/stale": "new", + "/d3/fresh": "1", + }) + require.NotNil(t, snap.LastReconcileDrift) + assert.EqualValues(t, 1, snap.LastReconcileDrift.OrphanKeys, + "the post-cutover-deleted key must be counted and pruned as an orphan") + }) + + t.Run("ExcludePrefixes", func(t *testing.T) { + cfg := baseCfg("/m4/", "/d4/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwriteAndPrune + cfg.ExcludePrefixes = []string{"/m4/skip/"} + _, err := src.Put(ctx, "/m4/keep/a", "1") + require.NoError(t, err) + _, err = src.Put(ctx, "/m4/skip/b", "2") + require.NoError(t, err) + // Pre-existing target key under the excluded image: never pruned. + _, err = dst.Put(ctx, "/d4/skip/old", "not-an-orphan") + require.NoError(t, err) + + r := startAgent(t, cfg, src, dst) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + waitTargetData(t, dst, cfg, 10*time.Second, map[string]string{ + "/d4/keep/a": "1", + "/d4/skip/old": "not-an-orphan", + }) + // A live write under the excluded prefix must not arrive either. + _, err = src.Put(ctx, "/m4/skip/c", "3") + require.NoError(t, err) + _, err = src.Put(ctx, "/m4/keep/d", "4") + require.NoError(t, err) + waitTargetData(t, dst, cfg, 10*time.Second, map[string]string{ + "/d4/keep/a": "1", + "/d4/keep/d": "4", + "/d4/skip/old": "not-an-orphan", + }) + }) +} + +// TestDrain covers scenario 9: with a quiesced source and mode Drain, the +// engine drains, verifies, reports a stable drained revision, and flips the +// fence role to Primary. +func TestDrain(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.Mode = mirroragent.ModeDrain + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 12) + r0 := sourceRevision(t, src, cfg.SourcePrefix) + + r := startAgent(t, cfg, src, dst) + runErr := r.waitErr(t, 30*time.Second) + require.NoError(t, runErr, "a completed drain returns nil") + + snap := r.agent.Snapshot() + assert.Equal(t, mirroragent.PhaseDrained, snap.Phase) + assert.True(t, snap.CutoverReady) + require.NotNil(t, snap.Cutover) + assert.Equal(t, r0, snap.Cutover.DrainTargetRevision) + assert.GreaterOrEqual(t, snap.Cutover.DrainedRevision, snap.Cutover.DrainTargetRevision) + assert.EqualValues(t, 12, snap.Cutover.SourceKeyCount) + assert.EqualValues(t, 12, snap.Cutover.TargetKeyCount) + assert.False(t, snap.Cutover.VerifiedTime.IsZero()) + + waitTargetData(t, dst, cfg, 5*time.Second, want) + f, _ := readFence(t, dst, cfg) + assert.Equal(t, mirroragent.RolePrimary, f.Role, "drain completion must flip the fence to Primary") + assert.Equal(t, snap.Cutover.DrainedRevision, f.Watermark) + + // Stability: the reported drained revision does not move. + time.Sleep(300 * time.Millisecond) + again := r.agent.Snapshot() + assert.Equal(t, snap.Cutover.DrainedRevision, again.Cutover.DrainedRevision) +} + +// TestTargetQuotaExhausted covers scenario 10: a target with an exhausted +// backend quota yields the typed TargetQuotaExhausted classification and the +// engine parks on the flat probe interval instead of hot-retrying. +func TestTargetQuotaExhausted(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, func(c *embed.Config) { c.QuotaBackendBytes = 4 * 1024 * 1024 }) + cfg := baseCfg("/src/", "/dst/") + cfg.QuotaProbeInterval = 250 * time.Millisecond + + // Fill the target past its quota until NOSPACE trips. + big := strings.Repeat("x", 1<<20) + sawNoSpace := false + for i := range 40 { + _, err := dst.Put(t.Context(), fmt.Sprintf("/fill/%02d", i), big) + if err != nil && errors.Is(rpctypes.Error(err), rpctypes.ErrNoSpace) { + sawNoSpace = true + break + } + require.NoError(t, err) + } + require.True(t, sawNoSpace, "target never hit NOSPACE while filling") + + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 2) + countingDst := &countingClient{Client: dst} + r := startAgent(t, cfg, src, countingDst) + + snap := waitSnap(t, r.agent, 20*time.Second, "quota classification", + func(s mirroragent.Snapshot) bool { return s.QuotaExhausted }) + assert.Equal(t, mirroragent.ClassQuota, snap.LastErrorClass, + "NOSPACE must classify as the quota class, never throttling or transient") + assert.Equal(t, mirroragent.PhaseDegraded, snap.Phase) + + // No hot retry loop: attempts are paced by QuotaProbeInterval. + before := countingDst.txns.Load() + time.Sleep(1200 * time.Millisecond) + delta := countingDst.txns.Load() - before + assert.LessOrEqual(t, delta, int64(8), + "quota parking must probe on the flat interval, not spin (saw %d Txns in 1.2s)", delta) + + // Run must still be parked, not returned. + select { + case err := <-r.done: + t.Fatalf("Run returned during quota park: %v", err) + default: + } +} diff --git a/pkg/mirroragent/reconcile.go b/pkg/mirroragent/reconcile.go new file mode 100644 index 00000000..ee440271 --- /dev/null +++ b/pkg/mirroragent/reconcile.go @@ -0,0 +1,419 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "strings" + "time" + + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// reconcilePass is the shared diff-and-repair pass: the OverwriteAndPrune +// genesis pass, the mandatory mark-and-sweep after any forced resync, the +// periodic pass when Config.ReconcileInterval enables it, and the Drain +// verification repair. It merges bounded pages of BOTH sides in +// key order — the source via the excluded-range-elided scan windows (so +// excluded data is never transferred, matching the genesis scan), the target +// one page at a time — so agent memory stays bounded by two pages no matter +// how divergent the target is (an orphan-heavy target is exactly the state +// this pass exists to repair). Repair puts source-truth values over missing +// or value-divergent target keys, deleteOrphans removes target keys with no +// source counterpart. The reserved checkpoint key and images of excluded +// prefixes are never touched, a sibling link's fence key aborts the pass +// (PrefixConflictError) instead of being pruned, and every repair/delete +// rides the same fenced Txn path as applies. cancelWatchWhenPaced is set by +// callers running while the live source watch is open and unread (see +// maybeCancelWatchForPacedRepair). On success the pass publishes its counts, +// completion time, and drift in one Snapshot update. +func (a *Agent) reconcilePass(ctx context.Context, repair, deleteOrphans, cancelWatchWhenPaced bool) error { + drift := Drift{Repaired: repair || deleteOrphans} + dstStart, dstEnd := a.rw.destRange() + b := newBatcher(a.cfg.MaxTxnOps, a.cfg.TxnFlushBytes) + dstCursor := dstStart + var srcSeen, dstSeen int64 + var pacedOps int + pager := &sourcePager{a: a, ranges: a.rw.scanRanges()} + for { + spage, more, err := pager.next(ctx) + if err != nil { + return err + } + // The window of target keys this source page is authoritative for; + // the last page's window swallows the tail of the destination range. + winEnd := dstEnd + if more && len(spage) > 0 { + winEnd = nextKey(a.rw.image(string(spage[len(spage)-1].Key))) + } + expected := make(map[string]string, len(spage)) + for _, kv := range spage { + if dstKey, ok := a.rw.rewrite(string(kv.Key)); ok { + expected[dstKey] = string(kv.Value) + srcSeen++ + } + } + // Stream the target side of the window one page at a time — never + // materialized whole. + tcursor := dstCursor + for { + tresp, terr := a.getRetry(ctx, a.dst, tcursor, clientv3.WithRange(winEnd), + clientv3.WithLimit(int64(a.cfg.PageKeyLimit)), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) + if terr != nil { + return terr + } + var ops []kvOp + for _, kv := range tresp.Kvs { + k := string(kv.Key) + if k == a.cfg.CheckpointKey || a.rw.excludedImage(k) { + continue + } + dstSeen++ + want, ok := expected[k] + if !ok { + if cerr := a.checkForeignFence(k, kv.Value); cerr != nil { + return cerr + } + drift.OrphanKeys++ + if deleteOrphans { + ops = append(ops, kvOp{key: k, isDelete: true}) + } + continue + } + delete(expected, k) + if want != string(kv.Value) { + drift.DivergentKeys++ + if repair { + ops = append(ops, kvOp{key: k, value: want}) + } + } + } + if err := a.enqueueRepairs(ctx, b, ops); err != nil { + return err + } + pacedOps += len(ops) + a.maybeCancelWatchForPacedRepair(cancelWatchWhenPaced, pacedOps) + if len(tresp.Kvs) == 0 || !tresp.More { + break + } + tcursor = nextKey(string(tresp.Kvs[len(tresp.Kvs)-1].Key)) + } + // Source keys of this window with no target counterpart. + var missing []kvOp + for k, v := range expected { + drift.MissingKeys++ + if repair { + missing = append(missing, kvOp{key: k, value: v}) + } + } + if err := a.enqueueRepairs(ctx, b, missing); err != nil { + return err + } + pacedOps += len(missing) + a.maybeCancelWatchForPacedRepair(cancelWatchWhenPaced, pacedOps) + dstCursor = winEnd + if !more { + break + } + } + if fs := b.flush(); fs != nil { + if err := a.applyRepairFlush(ctx, fs); err != nil { + return err + } + } + // Publish the pass outcome in ONE update: the per-side key counts as + // observed by this pass (pre-repair — the equality signal behind the + // InvariantsHeld condition), the completion timestamp, and the drift. + // The API contract binds the reconciliation time and drift as one + // record, so a concurrent Snapshot must never pair this pass's + // counts/timestamp with a previous pass's drift. + a.update(func(s *Snapshot) { + s.SourceKeyCount = srcSeen + s.TargetKeyCount = dstSeen + s.LastReconcileTime = time.Now() + s.LastReconcileDrift = &drift + }) + return nil +} + +// pacedRepairSecondsBeforeWatchCancel is how many seconds' worth of +// MaxOpsPerSecond pacing a diff pass may queue as repairs before the live +// source watch is cancelled (maybeCancelWatchForPacedRepair). +const pacedRepairSecondsBeforeWatchCancel = 3 + +// maybeCancelWatchForPacedRepair bounds clientv3's unbounded per-watcher +// buffer during a diff pass that runs while the live source watch is open +// and UNREAD — the periodic pass and the drain repair, both inline in +// consume; the genesis sweep's watch drains into the byte-bounded replay +// buffer instead and passes enabled=false. applyOps paces every repair +// flush by MaxOpsPerSecond, so a healthy pass repairing D keys leaves the +// watch unread for ~D/MaxOpsPerSecond seconds while the source keeps +// writing — a stall the backoff-driven cancels in applyFenced never see. +// Once the queued repairs exceed a few seconds of pacing, cancel the watch: +// the tail re-watches from the checkpoint watermark (the pass never moves +// it), and a compacted resume revision escalates to a forced resync whose +// mandatory sweep supersedes the pass anyway. +func (a *Agent) maybeCancelWatchForPacedRepair(enabled bool, pacedOps int) { + if !enabled || a.cfg.MaxOpsPerSecond <= 0 || + pacedOps <= pacedRepairSecondsBeforeWatchCancel*a.cfg.MaxOpsPerSecond { + return + } + a.cancelSourceWatch() +} + +// enqueueRepairs pushes repair/prune ops through the shared batcher, +// applying any flush sets that become due. +func (a *Agent) enqueueRepairs(ctx context.Context, b *batcher, ops []kvOp) error { + for _, op := range ops { + g := revGroup{rev: a.fence.Watermark, ops: []kvOp{op}} + for _, fs := range b.add(g) { + if err := a.applyRepairFlush(ctx, &fs); err != nil { + return err + } + } + } + return nil +} + +// checkForeignFence refuses to treat another EtcdMirror link's reserved +// checkpoint/fence key (recognized by the \x00-after-prefix reserved-key +// convention plus a decodable fence document) as a prunable orphan: +// overlapping destination prefixes must stop loudly instead of silently +// destroying the sibling link's fence and data. +func (a *Agent) checkForeignFence(key string, value []byte) error { + if !strings.Contains(key, "\x00") { + return nil + } + fv, err := DecodeFenceValue(value) + if err != nil || fv.LinkUID == a.cfg.LinkUID { + return nil + } + return &PrefixConflictError{ + Key: RedactKey(a.cfg.EffectiveDestPrefix(), []byte(key)), + OwnerLinkUID: fv.LinkUID, + } +} + +// sourcePager yields the source side of the reconcile merge one bounded page +// at a time, walking the excluded-range-elided scan windows in key order so +// excluded data is never transferred (server-side elision, matching the +// genesis scan). The reported more flag is false only on the final page of +// the final non-empty window; a one-page lookahead across window boundaries +// decides it. +type sourcePager struct { + a *Agent + ranges []scanRange + ri int + cursor string // resume point within ranges[ri]; "" means the range start + buf *srcPage +} + +type srcPage struct { + kvs []*mvccpb.KeyValue + more bool // resp.More within the page's own window +} + +func (p *sourcePager) next(ctx context.Context) ([]*mvccpb.KeyValue, bool, error) { + cur := p.buf + p.buf = nil + if cur == nil { + var err error + if cur, err = p.fetch(ctx); err != nil { + return nil, false, err + } + } + if cur == nil { + return nil, false, nil + } + if cur.more { + return cur.kvs, true, nil + } + // Window boundary: look one page ahead to decide whether any source key + // remains in a later window. + nxt, err := p.fetch(ctx) + if err != nil { + return nil, false, err + } + p.buf = nxt + return cur.kvs, nxt != nil, nil +} + +// fetch returns the next non-empty page across the remaining windows, or nil +// when every window is exhausted. +func (p *sourcePager) fetch(ctx context.Context) (*srcPage, error) { + for p.ri < len(p.ranges) { + kr := p.ranges[p.ri] + start := kr.start + if p.cursor != "" { + start = p.cursor + } + resp, err := p.a.getRetry(ctx, p.a.src, start, clientv3.WithRange(kr.end), + clientv3.WithLimit(int64(p.a.cfg.PageKeyLimit)), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) + if err != nil { + return nil, err + } + if resp.More { + p.cursor = nextKey(string(resp.Kvs[len(resp.Kvs)-1].Key)) + } else { + p.ri++ + p.cursor = "" + } + if len(resp.Kvs) > 0 { + return &srcPage{kvs: resp.Kvs, more: resp.More}, nil + } + } + return nil, nil +} + +// recordKeyCounts publishes the per-side in-scope key counts observed by a +// count-only drain verification, timestamped as a pass completion — the +// freshness input to the controller's InvariantsHeld condition. It never +// touches LastReconcileDrift (count equality cannot attest DivergentKeys); +// full diff passes publish counts, timestamp, and drift atomically inside +// reconcilePass instead. completeDrain stamps counts BEFORE its equality +// check, so a failed drain leaves a fresh timestamp beside unequal counts: +// freshness alone never attests a healthy pass. +func (a *Agent) recordKeyCounts(srcN, dstN int64) { + a.update(func(s *Snapshot) { + s.SourceKeyCount = srcN + s.TargetKeyCount = dstN + s.LastReconcileTime = time.Now() + }) +} + +// reconcileDue reports whether the periodic pass should run at now: never +// when disabled (ReconcileInterval <= 0), before the tail armed the deadline, +// before the deadline itself, or once a drain is requested — the drain's own +// verification pass (repair + prune + recount) supersedes the periodic pass. +func (a *Agent) reconcileDue(now time.Time) bool { + return a.cfg.ReconcileInterval > 0 && + !a.nextReconcile.IsZero() && + !now.Before(a.nextReconcile) && + a.cfg.Mode != ModeDrain && + !a.drainReq.Load() +} + +// scheduleNextReconcile pushes the periodic deadline one full interval out +// from now; a no-op when the periodic pass is disabled. Mandatory sweeps call +// this too — they just produced the same signal, so re-running the periodic +// pass sooner adds cost without information. +func (a *Agent) scheduleNextReconcile() { + if a.cfg.ReconcileInterval <= 0 { + return + } + a.nextReconcile = time.Now().Add(a.cfg.ReconcileInterval) +} + +// maybeReconcile runs one periodic diff-and-repair pass when due, inline on +// the Run goroutine — never concurrently with a genesis scan (consume is not +// running), a forced-resync sweep (same), a drain (gated), or itself. A +// drain-gated fire still re-arms the deadline so the tail's timer never spins +// on a stale one. Error flow: transient, throttle, and quota errors are +// absorbed INSIDE the pass by getRetry/applyFenced — the pass blocks until +// they heal or ctx cancels (a wedged pass cannot be preempted by a drain +// check) — and no pass read is revision-pinned, so what actually escapes +// through consume → tail → Run is ClassPermanent (fails the agent) or ctx +// cancellation. tail's transient/Resync arms remain as taxonomy-consistent +// safety nets; the transient arm also recovers the watch-closed error after +// a pass cancelled the source watch (maybeCancelWatchForPacedRepair) by +// re-watching from the checkpoint watermark. +func (a *Agent) maybeReconcile(ctx context.Context) error { + if !a.reconcileDue(time.Now()) { + a.scheduleNextReconcile() + return nil + } + if err := a.reconcilePass(ctx, true, a.cfg.ReconcileDeleteOrphans, true); err != nil { + return err + } + a.scheduleNextReconcile() + return nil +} + +// applyRepairFlush writes repair/prune ops under the fence without moving +// any checkpoint field: the pass is positionless, only the compare matters. +func (a *Agent) applyRepairFlush(ctx context.Context, fs *flushSet) error { + f := a.fence + f.Epoch = a.cfg.Epoch + return a.applyOps(ctx, fs, f) +} + +// verifyCounts counts in-scope keys on both sides: excluded prefixes are +// elided via the scan windows and the reserved checkpoint key is not counted +// on the target. Source counts are pinned at the checkpoint watermark (the +// drained revision during a drain) so a non-quiesced source still yields a +// coherent snapshot; if that revision has been compacted the count falls +// back to an unpinned re-read. +func (a *Agent) verifyCounts(ctx context.Context) (srcN, dstN int64, err error) { + if srcN, err = a.countSourceInScope(ctx, a.watermark()); err != nil { + if Classify(err) != ClassResync { + return 0, 0, err + } + if srcN, err = a.countSourceInScope(ctx, 0); err != nil { + return 0, 0, err + } + } + dstStart, dstEnd := a.rw.destRange() + if dstN, err = a.countRange(ctx, a.dst, dstStart, dstEnd, 0); err != nil { + return 0, 0, err + } + ck, err := a.getRetry(ctx, a.dst, a.cfg.CheckpointKey, clientv3.WithCountOnly()) + if err != nil { + return 0, 0, err + } + dstN -= ck.Count + for _, p := range a.cfg.ExcludePrefixes { + if !strings.HasPrefix(p, a.cfg.SourcePrefix) { + continue + } + s, e := keyRange(a.rw.image(p)) + n, cerr := a.countRange(ctx, a.dst, s, e, 0) + if cerr != nil { + return 0, 0, cerr + } + dstN -= n + } + return srcN, dstN, nil +} + +// countSourceInScope sums the in-scope source key count over the scan +// windows, pinned at rev when rev > 0. +func (a *Agent) countSourceInScope(ctx context.Context, rev int64) (int64, error) { + var total int64 + for _, kr := range a.rw.scanRanges() { + n, err := a.countRange(ctx, a.src, kr.start, kr.end, rev) + if err != nil { + return 0, err + } + total += n + } + return total, nil +} + +func (a *Agent) countRange(ctx context.Context, cl Client, start, end string, rev int64) (int64, error) { + opts := []clientv3.OpOption{clientv3.WithRange(end), clientv3.WithCountOnly()} + if rev > 0 { + opts = append(opts, clientv3.WithRev(rev)) + } + resp, err := a.getRetry(ctx, cl, start, opts...) + if err != nil { + return 0, err + } + return resp.Count, nil +} diff --git a/pkg/mirroragent/reconcile_test.go b/pkg/mirroragent/reconcile_test.go new file mode 100644 index 00000000..751c72ab --- /dev/null +++ b/pkg/mirroragent/reconcile_test.go @@ -0,0 +1,125 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestReconcileDue: the periodic pass's pure gating logic — disabled +// interval, unarmed deadline, a not-yet-due deadline, and a requested drain +// (whose own verification pass supersedes the periodic one) all veto it. +func TestReconcileDue(t *testing.T) { + now := time.Now() + cases := []struct { + name string + make func() *Agent + want bool + }{ + {name: "due", make: func() *Agent { + a := &Agent{cfg: Config{ReconcileInterval: time.Minute, Mode: ModeSync}} + a.nextReconcile = now.Add(-time.Second) + return a + }, want: true}, + {name: "due exactly at deadline", make: func() *Agent { + a := &Agent{cfg: Config{ReconcileInterval: time.Minute, Mode: ModeSync}} + a.nextReconcile = now + return a + }, want: true}, + {name: "disabled interval", make: func() *Agent { + a := &Agent{cfg: Config{Mode: ModeSync}} + a.nextReconcile = now.Add(-time.Second) + return a + }, want: false}, + {name: "deadline not armed", make: func() *Agent { + return &Agent{cfg: Config{ReconcileInterval: time.Minute, Mode: ModeSync}} + }, want: false}, + {name: "before deadline", make: func() *Agent { + a := &Agent{cfg: Config{ReconcileInterval: time.Minute, Mode: ModeSync}} + a.nextReconcile = now.Add(time.Second) + return a + }, want: false}, + {name: "drain mode", make: func() *Agent { + a := &Agent{cfg: Config{ReconcileInterval: time.Minute, Mode: ModeDrain}} + a.nextReconcile = now.Add(-time.Second) + return a + }, want: false}, + {name: "drain requested", make: func() *Agent { + a := &Agent{cfg: Config{ReconcileInterval: time.Minute, Mode: ModeSync}} + a.nextReconcile = now.Add(-time.Second) + a.drainReq.Store(true) + return a + }, want: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, tc.make().reconcileDue(now)) + }) + } +} + +// TestScheduleNextReconcile: a no-op when disabled; otherwise the deadline +// lands one interval out, and re-scheduling pushes it further (the +// mandatory-sweep re-arm semantics — a sweep just produced the same signal). +func TestScheduleNextReconcile(t *testing.T) { + disabled := &Agent{cfg: Config{}} + disabled.scheduleNextReconcile() + assert.True(t, disabled.nextReconcile.IsZero(), "disabled scheduler must not arm a deadline") + + a := &Agent{cfg: Config{ReconcileInterval: time.Hour}} + before := time.Now() + a.scheduleNextReconcile() + first := a.nextReconcile + assert.False(t, first.Before(before.Add(time.Hour)), "deadline must be at least one interval out") + assert.False(t, first.After(time.Now().Add(time.Hour)), "deadline must be at most one interval out") + + a.scheduleNextReconcile() + assert.False(t, a.nextReconcile.Before(first), "re-scheduling must push the deadline out") +} + +// TestMaybeCancelWatchForPacedRepair: the paced-repair watch cancel fires +// only for a pass that enabled it (not the genesis sweep, whose watch drains +// into the replay buffer), only under a MaxOpsPerSecond pacer, and only past +// the threshold's worth of queued repair ops. +func TestMaybeCancelWatchForPacedRepair(t *testing.T) { + const rate = 100 + threshold := pacedRepairSecondsBeforeWatchCancel * rate + cases := []struct { + name string + enabled bool + maxOps int + ops int + want bool + }{ + {name: "past threshold", enabled: true, maxOps: rate, ops: threshold + 1, want: true}, + {name: "at threshold", enabled: true, maxOps: rate, ops: threshold, want: false}, + {name: "disabled for the pass", enabled: false, maxOps: rate, ops: threshold + 1, want: false}, + {name: "unpaced config", enabled: true, maxOps: 0, ops: threshold + 1, want: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + a := &Agent{cfg: Config{MaxOpsPerSecond: tc.maxOps}} + var cancelled bool + a.watchCancel = func() { cancelled = true } + a.maybeCancelWatchForPacedRepair(tc.enabled, tc.ops) + assert.Equal(t, tc.want, cancelled) + }) + } +} diff --git a/pkg/mirroragent/rewrite.go b/pkg/mirroragent/rewrite.go new file mode 100644 index 00000000..4a67c0c0 --- /dev/null +++ b/pkg/mirroragent/rewrite.go @@ -0,0 +1,184 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "strings" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +// rewriter implements the mirror's single key-rewrite formula: +// +// key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix) +// +// It is an anchored strip-and-reprefix — never a substring replace — and it +// is order-preserving within the source scope, which the prune pass's merge +// scan relies on. +type rewriter struct { + srcPrefix string + // dstPrefix is the effective destination prefix (target.prefix + + // destPrefix). + dstPrefix string + exclude []string + // reservedKey is never produced by any apply path, even if a source key + // happens to map onto it. + reservedKey string +} + +func newRewriter(cfg Config) *rewriter { + return &rewriter{ + srcPrefix: cfg.SourcePrefix, + dstPrefix: cfg.EffectiveDestPrefix(), + exclude: cfg.ExcludePrefixes, + reservedKey: cfg.CheckpointKey, + } +} + +// inScope reports whether a source key is mirrored: under the source prefix +// and not excluded. +func (r *rewriter) inScope(srcKey string) bool { + if !strings.HasPrefix(srcKey, r.srcPrefix) { + return false + } + return !r.excluded(srcKey) +} + +// excluded reports whether a source key falls under any exclude prefix. +func (r *rewriter) excluded(srcKey string) bool { + for _, p := range r.exclude { + if strings.HasPrefix(srcKey, p) { + return true + } + } + return false +} + +// rewrite maps an in-scope source key to its target key. ok is false when +// the key is out of scope, excluded, or would collide with the reserved +// checkpoint key. +func (r *rewriter) rewrite(srcKey string) (string, bool) { + if !r.inScope(srcKey) { + return "", false + } + dst := r.dstPrefix + strings.TrimPrefix(srcKey, r.srcPrefix) + if dst == r.reservedKey { + return "", false + } + return dst, true +} + +// image maps ANY source key under the source prefix (including excluded +// ones) to its would-be target key, for cursor/window arithmetic in the +// merge scan. Callers must ensure srcKey has the source prefix. +func (r *rewriter) image(srcKey string) string { + return r.dstPrefix + strings.TrimPrefix(srcKey, r.srcPrefix) +} + +// excludedImage reports whether a TARGET key falls under the image of an +// exclude prefix: such keys are never treated as orphans by prune passes. +func (r *rewriter) excludedImage(dstKey string) bool { + for _, p := range r.exclude { + if !strings.HasPrefix(p, r.srcPrefix) { + continue // excluded range is outside the mirrored scope + } + if strings.HasPrefix(dstKey, r.image(p)) { + return true + } + } + return false +} + +// sourceRange returns the [start, end) watch/scan range for the source +// prefix. An empty prefix means the whole keyspace. +func (r *rewriter) sourceRange() (string, string) { + return keyRange(r.srcPrefix) +} + +// scanRange is one [Start, End) source read window. End == rangeEndInf is +// etcd's ">= Start" sentinel (unbounded). +type scanRange struct { + start, end string +} + +// rangeEndInf is etcd's unbounded range-end sentinel ("all keys >= start"). +const rangeEndInf = "\x00" + +// endAfter reports whether range end e lies strictly after key k, treating +// the sentinel as +inf. +func endAfter(e, k string) bool { + return e == rangeEndInf || e > k +} + +// scanRanges returns the source range minus every excluded range: the +// sorted, disjoint windows the genesis scan reads. Range Gets are issued +// ONLY over these windows, so excluded data is elided server-side — never +// transferred and filtered client-side. (The watch still spans the whole +// source range: watches cannot be decomposed without multiplying streams, +// and its events are filtered through rewrite.) +func (r *rewriter) scanRanges() []scanRange { + start, end := r.sourceRange() + out := []scanRange{{start: start, end: end}} + for _, p := range r.exclude { + es, ee := keyRange(p) + next := make([]scanRange, 0, len(out)+1) + for _, w := range out { + next = append(next, subtractRange(w, es, ee)...) + } + out = next + } + return out +} + +// subtractRange removes the [es, ee) slice from window w, yielding 0, 1, or +// 2 remaining windows. +func subtractRange(w scanRange, es, ee string) []scanRange { + // No overlap: the excluded range ends at/before the window starts, or + // starts at/after the window ends. + if !endAfter(ee, w.start) || !endAfter(w.end, es) { + return []scanRange{w} + } + out := make([]scanRange, 0, 2) + if es > w.start { + out = append(out, scanRange{start: w.start, end: es}) + } + if ee != rangeEndInf && endAfter(w.end, ee) { + out = append(out, scanRange{start: ee, end: w.end}) + } + return out +} + +// destRange returns the [start, end) range covering the effective +// destination prefix on the target. +func (r *rewriter) destRange() (string, string) { + return keyRange(r.dstPrefix) +} + +// keyRange converts a prefix to an etcd [start, end) range. The empty prefix +// maps to the whole keyspace ("\x00" with the >=-key range end "\x00"). +func keyRange(prefix string) (string, string) { + if prefix == "" { + return "\x00", "\x00" + } + return prefix, clientv3.GetPrefixRangeEnd(prefix) +} + +// nextKey returns the smallest key strictly greater than k, for cursor +// advancement. +func nextKey(k string) string { + return k + "\x00" +} diff --git a/pkg/mirroragent/rewrite_test.go b/pkg/mirroragent/rewrite_test.go new file mode 100644 index 00000000..e43e641c --- /dev/null +++ b/pkg/mirroragent/rewrite_test.go @@ -0,0 +1,157 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRewriteFormula pins the single anchored rewrite formula: +// key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix). +func TestRewriteFormula(t *testing.T) { + cases := []struct { + name string + cfg Config + srcKey string + want string + wantOK bool + }{ + {name: "default destPrefix strips the source prefix", + cfg: Config{SourcePrefix: "/apps/", TargetPrefix: "/mirror/"}, + srcKey: "/apps/foo", want: "/mirror/foo", wantOK: true}, + {name: "non-empty destPrefix is the middle term", + cfg: Config{SourcePrefix: "/apps/", TargetPrefix: "/mirror/", DestPrefix: "west/"}, + srcKey: "/apps/foo", want: "/mirror/west/foo", wantOK: true}, + {name: "key equal to the source prefix maps to the effective dest prefix", + cfg: Config{SourcePrefix: "/apps/", TargetPrefix: "/mirror/", DestPrefix: "west/"}, + srcKey: "/apps/", want: "/mirror/west/", wantOK: true}, + {name: "anchored, never a substring replace", + cfg: Config{SourcePrefix: "/apps/", TargetPrefix: "/mirror/"}, + srcKey: "/apps/foo/apps/bar", want: "/mirror/foo/apps/bar", wantOK: true}, + {name: "out of scope", + cfg: Config{SourcePrefix: "/apps/", TargetPrefix: "/mirror/"}, + srcKey: "/other/foo", wantOK: false}, + {name: "excluded prefix", + cfg: Config{SourcePrefix: "/apps/", TargetPrefix: "/mirror/", + ExcludePrefixes: []string{"/apps/skip/"}}, + srcKey: "/apps/skip/foo", wantOK: false}, + {name: "empty source prefix mirrors the whole keyspace", + cfg: Config{TargetPrefix: "/mirror/"}, + srcKey: "/anything", want: "/mirror//anything", wantOK: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rw := newRewriter(tc.cfg.withDefaults()) + got, ok := rw.rewrite(tc.srcKey) + require.Equal(t, tc.wantOK, ok) + if tc.wantOK { + assert.Equal(t, tc.want, got) + } + }) + } +} + +// TestRewriteReservedKeyCollision: a source key whose image would be the +// reserved checkpoint key is never produced by any apply path. +func TestRewriteReservedKeyCollision(t *testing.T) { + cfg := Config{SourcePrefix: "/s/", TargetPrefix: "/d/", CheckpointKey: "/d/ckpt"}.withDefaults() + rw := newRewriter(cfg) + + _, ok := rw.rewrite("/s/ckpt") + assert.False(t, ok, "the reserved key's preimage must be dropped") + got, ok := rw.rewrite("/s/ckpt2") + assert.True(t, ok, "only the exact match is reserved") + assert.Equal(t, "/d/ckpt2", got) +} + +func TestExcludedImage(t *testing.T) { + cfg := Config{ + SourcePrefix: "/s/", + TargetPrefix: "/d/", + ExcludePrefixes: []string{"/s/skip/", "/elsewhere/"}, + }.withDefaults() + rw := newRewriter(cfg) + + assert.True(t, rw.excludedImage("/d/skip/x"), "images of excluded ranges are never orphans") + assert.False(t, rw.excludedImage("/d/keep/x")) + assert.False(t, rw.excludedImage("/d/elsewhere/x"), + "excludes outside the mirrored scope have no image") +} + +func TestKeyRange(t *testing.T) { + s, e := keyRange("/p/") + assert.Equal(t, "/p/", s) + assert.Equal(t, "/p0", e, "prefix range end increments the last byte") + + s, e = keyRange("") + assert.Equal(t, "\x00", s, "empty prefix means the whole keyspace") + assert.Equal(t, "\x00", e, "with the >=-key sentinel range end") +} + +// TestScanRanges pins the range decomposition: the genesis scan reads ONLY +// the source range minus the excluded ranges — excluded data is elided +// server-side, not filtered client-side. +func TestScanRanges(t *testing.T) { + cases := []struct { + name string + src string + exclude []string + want []scanRange + }{ + {name: "no excludes", src: "/s/", + want: []scanRange{{"/s/", "/s0"}}}, + {name: "one middle exclude", src: "/s/", exclude: []string{"/s/b/"}, + want: []scanRange{{"/s/", "/s/b/"}, {"/s/b0", "/s0"}}}, + {name: "two excludes stay sorted", src: "/s/", exclude: []string{"/s/b/", "/s/d/"}, + want: []scanRange{{"/s/", "/s/b/"}, {"/s/b0", "/s/d/"}, {"/s/d0", "/s0"}}}, + {name: "exclude at the start", src: "/s/", exclude: []string{"/s/"}, + want: []scanRange{}}, + {name: "exclude covering the whole source prefix", src: "/s/sub/", exclude: []string{"/s/"}, + want: []scanRange{}}, + {name: "exclude outside the source range", src: "/s/", exclude: []string{"/t/"}, + want: []scanRange{{"/s/", "/s0"}}}, + {name: "whole keyspace with one exclude", src: "", exclude: []string{"/b/"}, + want: []scanRange{{"\x00", "/b/"}, {"/b0", "\x00"}}}, + {name: "nested excludes merge", src: "/s/", exclude: []string{"/s/b/", "/s/b/c/"}, + want: []scanRange{{"/s/", "/s/b/"}, {"/s/b0", "/s0"}}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rw := newRewriter(Config{ + SourcePrefix: tc.src, + TargetPrefix: "/d/", + ExcludePrefixes: tc.exclude, + }.withDefaults()) + got := rw.scanRanges() + if len(tc.want) == 0 { + assert.Empty(t, got) + return + } + require.Equal(t, tc.want, got) + }) + } +} + +func TestEndAfter(t *testing.T) { + assert.True(t, endAfter(rangeEndInf, "/z/"), "the sentinel end is +inf") + assert.True(t, endAfter("/b/", "/a/")) + assert.False(t, endAfter("/a/", "/a/")) + assert.False(t, endAfter("/a/", "/b/")) +} diff --git a/pkg/mirroragent/scan.go b/pkg/mirroragent/scan.go new file mode 100644 index 00000000..70d20b5f --- /dev/null +++ b/pkg/mirroragent/scan.go @@ -0,0 +1,431 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "fmt" + "sync" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +// scanRestartError aborts a genesis attempt so Run restarts the scan from a +// fresh R0 — a bounded retry, distinct from a forced resync (the checkpoint +// is not invalidated). Buffered watch events were dropped, so the restarted +// scan owes a mark-and-sweep prune (deletes from the dropped window must not +// resurrect). +type scanRestartError struct { + Cause ScanRestartCause + Err error +} + +func (e *scanRestartError) Error() string { + return fmt.Sprintf("genesis scan restart required (%s): %v", e.Cause, e.Err) +} +func (e *scanRestartError) Unwrap() error { return e.Err } + +// replayBuffer drains the watch opened before a genesis scan into a +// byte-bounded buffer, so the reflector replay base is bounded by +// Config.WatchBufferBytes instead of clientv3's unbounded internal queue. +// On overflow — or a watch reconnect landing below the source compact +// revision — it cancels the watch and records a scanRestartError. +type replayBuffer struct { + limit int64 + cancelWatch context.CancelFunc + + mu sync.Mutex + resps []clientv3.WatchResponse + bytes int64 + fail *scanRestartError + + stopc chan struct{} + donec chan struct{} +} + +func newReplayBuffer(limit int64, cancelWatch context.CancelFunc) *replayBuffer { + return &replayBuffer{ + limit: limit, + cancelWatch: cancelWatch, + stopc: make(chan struct{}), + donec: make(chan struct{}), + } +} + +// fill consumes wch until stop is called, the channel dies, or a restart +// condition hits. Run it in a goroutine; it never blocks the scan. +func (rb *replayBuffer) fill(wch clientv3.WatchChan) { + defer close(rb.donec) + for { + select { + case <-rb.stopc: + return + case wr, ok := <-wch: + if !ok { + // Channel died (transient watch failure): keep what was + // buffered; the tail re-watches from the watermark and the + // missed span comes from etcd's watch history. + return + } + if wr.CompactRevision != 0 { + rb.setFail(&scanRestartError{Cause: ScanRestartWatchCompactedMidScan, Err: wr.Err()}) + return + } + if wr.Err() != nil || wr.IsProgressNotify() { + // Errors surface via channel close; progress notifications + // carry nothing to replay (the watermark must stay the scan + // base R0 while scanning). + continue + } + var n int64 + for _, ev := range wr.Events { + n += int64(len(ev.Kv.Key) + len(ev.Kv.Value)) + } + rb.mu.Lock() + rb.bytes += n + over := rb.bytes > rb.limit + if !over { + rb.resps = append(rb.resps, wr) + } + rb.mu.Unlock() + if over { + rb.setFail(&scanRestartError{ + Cause: ScanRestartWatchBufferOverflow, + Err: fmt.Errorf("replay buffer exceeded %d bytes before the base scan completed", + rb.limit), + }) + return + } + } + } +} + +// setFail records the restart condition and cancels the watch so the client +// stops accumulating events for a doomed attempt. +func (rb *replayBuffer) setFail(e *scanRestartError) { + rb.mu.Lock() + rb.fail = e + rb.mu.Unlock() + rb.cancelWatch() +} + +// err returns the pending scanRestartError, if any. +func (rb *replayBuffer) err() error { + rb.mu.Lock() + defer rb.mu.Unlock() + if rb.fail != nil { + return rb.fail + } + return nil +} + +// stop halts filling and hands back the buffered responses for replay. +func (rb *replayBuffer) stop() []clientv3.WatchResponse { + close(rb.stopc) + <-rb.donec + rb.mu.Lock() + defer rb.mu.Unlock() + return rb.resps +} + +// genesis is the cold-start / forced-resync path: RequireEmpty gate, fence +// claim, watch-before-scan, unpinned chunked scan, optional prune pass, +// replay of the buffered watch events, then the live tail over the same +// watch channel. +func (a *Agent) genesis(ctx context.Context, st startState) error { + a.setPhase(PhaseInitialSync) + srcStart, srcEnd := a.rw.sourceRange() + + // A forced resync owes a mandatory mark-and-sweep, as does an + // OverwriteAndPrune genesis. The obligation is stamped into every + // checkpoint (FenceValue.PrunePending) until the prune completes, so a + // crash mid-genesis cannot silently drop the owed sweep. + if st.forced || a.cfg.InitialSyncMode == InitialSyncOverwriteAndPrune { + a.prunePending = true + } + + // Forced resyncs never re-check RequireEmpty: the fence proves the + // destination data is this link's own. A cluster-identity mismatch + // re-arms it. + checkEmpty := (!st.haveCheckpoint && !st.forced) || st.rearmEmpty + if a.cfg.InitialSyncMode == InitialSyncRequireEmpty && checkEmpty { + if err := a.requireEmpty(ctx); err != nil { + return err + } + } + + var r0 int64 + cursor := srcStart + subrev := int64(0) + if st.haveCheckpoint && st.scanning && st.scanCursor != "" { + // Resume an interrupted scan from its cursor at the recorded base. + r0 = st.watermark + cursor = nextKey(st.scanCursor) + subrev = st.subRevision + } else { + // Observe the scan base BEFORE scanning: the watch starts at r0+1 + // and buffered events replay over the scanned base. The windows' + // counts are the InitialSync denominator, for free (excluded + // prefixes are elided from the windows, so they are never counted). + rev, total, err := a.countScanRanges(ctx) + if err != nil { + return err + } + r0 = rev + a.update(func(s *Snapshot) { + s.InitialSyncTotalKeyCount = total + s.InitialSyncKeyCount = 0 + s.InitialSyncStartTime = time.Now() + s.InitialSyncCompletionTime = time.Time{} + s.LeaseBackedKeyCount = 0 + s.SourceRevision = r0 + }) + } + + // Claim (or refresh) the fence before any data write. + if err := a.applyFenced(ctx, nil, a.newFence(r0, true, st.scanCursor, subrev), "", 0); err != nil { + return err + } + + // Open the watch BEFORE the scan (reflector pattern): events during the + // scan buffer (byte-bounded) and replay over the scanned base + // afterwards, so mid-scan compaction cannot invalidate anything the + // scan needs — the scan itself reads unpinned, at the current revision. + wctx, wcancel := context.WithCancel(clientv3.WithRequireLeader(ctx)) + defer wcancel() + wch := a.src.Watch(wctx, srcStart, clientv3.WithRange(srcEnd), + clientv3.WithRev(r0+1), clientv3.WithProgressNotify()) + rb := newReplayBuffer(a.cfg.WatchBufferBytes, wcancel) + go rb.fill(wch) + // Publish the cancel so a sustained target stall during the scan or the + // replay can stop the source stream (the tail then re-watches from the + // watermark). rb.fill tolerates the resulting channel death by design. + a.watchCancel = wcancel + + if err := a.scan(ctx, cursor, r0, subrev, rb); err != nil { + return err + } + + // The mandatory mark-and-sweep: the OverwriteAndPrune genesis pass and + // every forced resync, cleared only once the pass completed. + if a.prunePending { + // No paced watch cancel: during genesis the watch drains into the + // byte-bounded replay buffer, so the pass cannot grow it unread. + if err := a.reconcilePass(ctx, true, true, false); err != nil { + return err + } + a.prunePending = false + // This sweep produced the same signal as a periodic pass; push the + // next periodic deadline a full interval out. + a.scheduleNextReconcile() + } + + // Scan complete: first revision-complete checkpoint at the scan base. + if err := a.applyFenced(ctx, nil, a.newFence(r0, false, "", 0), "", 0); err != nil { + return err + } + a.update(func(s *Snapshot) { + s.InitialSyncCompletionTime = time.Now() + if r0 > s.Watermark { + s.Watermark = r0 + } + s.LastProgressTime = time.Now() + }) + + // Hand the watch over from the buffer to the live tail: stop filling, + // replay what was buffered over the scanned base, then consume the + // still-open channel directly. A restart condition recorded at any + // point during scan/prune aborts the attempt instead. + buffered := rb.stop() + if err := rb.err(); err != nil { + return err + } + for i := range buffered { + // live=false: replay applies run INSIDE the (possibly forced-resync) + // genesis and must not reset the resync-loop detector. + if err := a.handleResponse(ctx, &buffered[i], false); err != nil { + return err + } + } + return a.tail(ctx, wch, wcancel, r0) +} + +// countScanRanges counts the in-scope source keys window by window and +// returns the revision R0 observed by the first (linearizable) read plus +// the total count. No Get ever spans an excluded range. +func (a *Agent) countScanRanges(ctx context.Context) (r0, total int64, err error) { + ranges := a.rw.scanRanges() + if len(ranges) == 0 { + // Everything is excluded: one point read still pins R0. + start, _ := a.rw.sourceRange() + resp, gerr := a.getRetry(ctx, a.src, start, clientv3.WithCountOnly()) + if gerr != nil { + return 0, 0, gerr + } + return resp.Header.Revision, 0, nil + } + for i, kr := range ranges { + resp, gerr := a.getRetry(ctx, a.src, kr.start, + clientv3.WithRange(kr.end), clientv3.WithCountOnly()) + if gerr != nil { + return 0, 0, gerr + } + if i == 0 { + r0 = resp.Header.Revision + } + total += resp.Count + } + return r0, total, nil +} + +// scan pulls byte-bounded pages at the current revision (never pinned) over +// the decomposed in-scope windows and applies them as fenced Txns carrying +// the in-scan checkpoint. rb is polled between pages so an overflowed or +// compacted replay buffer aborts the attempt promptly. +func (a *Agent) scan(ctx context.Context, cursor string, r0, subrev int64, rb *replayBuffer) error { + b := newBatcher(a.cfg.MaxTxnOps, a.cfg.TxnFlushBytes) + limit := a.cfg.PageKeyLimit + for _, kr := range a.rw.scanRanges() { + if !endAfter(kr.end, cursor) { + continue // window fully below the resume cursor + } + next := kr.start + if cursor > next { + next = cursor + } + var err error + if limit, subrev, err = a.scanWindow(ctx, next, kr.end, r0, limit, subrev, b, rb); err != nil { + return err + } + } + if fs := b.flush(); fs != nil { + subrev++ + return a.applyScanFlush(ctx, fs, subrev) + } + return rb.err() +} + +// scanWindow pages one [cursor, end) window through the shared batcher. +func (a *Agent) scanWindow( + ctx context.Context, cursor, end string, r0 int64, + limit int, subrev int64, b *batcher, rb *replayBuffer, +) (int, int64, error) { + for { + if err := rb.err(); err != nil { + return limit, subrev, err + } + resp, err := a.getRetry(ctx, a.src, cursor, clientv3.WithRange(end), + clientv3.WithLimit(int64(limit)), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) + if err != nil { + return limit, subrev, err + } + if len(resp.Kvs) == 0 { + return limit, subrev, nil + } + var pageBytes, leased int64 + for _, kv := range resp.Kvs { + pageBytes += int64(len(kv.Key) + len(kv.Value)) + dstKey, ok := a.rw.rewrite(string(kv.Key)) + if !ok { + continue + } + if kv.Lease != 0 { + leased++ + } + // Synthetic single-key groups: a snapshot has no per-key + // revision boundaries to preserve. + g := revGroup{rev: r0, ops: []kvOp{{ + key: dstKey, value: string(kv.Value), srcKey: string(kv.Key), + }}} + for _, fs := range b.add(g) { + subrev++ + if err := a.applyScanFlush(ctx, &fs, subrev); err != nil { + return limit, subrev, err + } + } + } + if leased > 0 { + a.update(func(s *Snapshot) { s.LeaseBackedKeyCount += leased }) + } + limit = adaptLimit(limit, a.cfg.PageKeyLimit, pageBytes, a.cfg.PageBytes) + if !resp.More { + return limit, subrev, nil + } + cursor = nextKey(string(resp.Kvs[len(resp.Kvs)-1].Key)) + } +} + +// applyScanFlush applies one scan flush set with the in-scan checkpoint +// (Scanning=true, cursor advanced, page ordinal in SubRevision) riding the +// same Txn, so a restarted agent resumes the scan instead of starting over. +func (a *Agent) applyScanFlush(ctx context.Context, fs *flushSet, subrev int64) error { + f := a.newFence(fs.watermark, true, fs.lastSrcKey, subrev) + if err := a.applyOps(ctx, fs, f); err != nil { + return err + } + a.update(func(s *Snapshot) { + s.InitialSyncKeyCount += int64(len(fs.ops)) + s.LastProgressTime = time.Now() + }) + return nil +} + +// requireEmpty enforces InitialSyncRequireEmpty over the effective +// destination prefix; the reserved checkpoint key is excluded by exact +// match. +func (a *Agent) requireEmpty(ctx context.Context) error { + start, end := a.rw.destRange() + resp, err := a.getRetry(ctx, a.dst, start, clientv3.WithRange(end), clientv3.WithCountOnly()) + if err != nil { + return err + } + n := resp.Count + if n > 0 { + ck, cerr := a.getRetry(ctx, a.dst, a.cfg.CheckpointKey, clientv3.WithCountOnly()) + if cerr != nil { + return cerr + } + n -= ck.Count + } + if n > 0 { + return &EmptyTargetViolationError{RangeStart: start, RangeEnd: end, KeyCount: n} + } + return nil +} + +// adaptLimit derives the next page's key limit from the observed bytes/key, +// enforcing the PageBytes bound etcd Range lacks natively. +func adaptLimit(cur, maxLimit int, gotBytes, maxBytes int64) int { + if gotBytes <= 0 { + return cur + } + switch { + case gotBytes > maxBytes && cur > 1: + cur /= 2 + if cur < 1 { + cur = 1 + } + case gotBytes*2 < maxBytes && cur < maxLimit: + cur *= 2 + if cur > maxLimit { + cur = maxLimit + } + } + return cur +} diff --git a/pkg/mirroragent/snapshot.go b/pkg/mirroragent/snapshot.go new file mode 100644 index 00000000..7c598cd1 --- /dev/null +++ b/pkg/mirroragent/snapshot.go @@ -0,0 +1,210 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import "time" + +// Phase mirrors EtcdMirrorPhase in api/v1alpha1 (the library adds Drained, +// which the controller maps to the CutoverReady condition). +type Phase string + +const ( + PhaseConnecting Phase = "Connecting" + PhaseInitialSync Phase = "InitialSync" + PhaseSyncing Phase = "Syncing" + PhaseDegraded Phase = "Degraded" + PhaseFailed Phase = "Failed" + // PhaseDrained means the drain completed, verification passed, and the + // fence role is Primary. Run has returned; the agent will never write + // again. + PhaseDrained Phase = "Drained" +) + +// ScanRestartCause says why a genesis scan attempt was aborted and restarted +// from a fresh R0. Both causes surface as one operator-facing event +// (InitialSyncCompactionRaced) with the cause named in the message, so +// "buffer too small for churn" is never conflated with "compaction won a +// race the design eliminates". Repeated restarts count toward the +// resync-loop detector. +type ScanRestartCause string + +const ( + // ScanRestartWatchBufferOverflow: the replay buffer exceeded + // Config.WatchBufferBytes before the base scan completed — a + // memory-bound retry, NOT a compaction race. + ScanRestartWatchBufferOverflow ScanRestartCause = "WatchBufferOverflow" + // ScanRestartWatchCompactedMidScan: a watch reconnect landed below the + // source compact revision while the scan was still running — the rare + // genuine race. + ScanRestartWatchCompactedMidScan ScanRestartCause = "WatchCompactedMidScan" +) + +// Drift is the outcome of one reconciliation pass. +type Drift struct { + // MissingKeys were present on the source but absent on the target + // (repaired when the pass repairs). + MissingKeys int64 `json:"missingKeys"` + // DivergentKeys were present on both sides with different values + // (repaired to source truth when the pass repairs). Distinct from + // MissingKeys so operators can tell "a resync dropped keys" from "a + // blind window went stale". + DivergentKeys int64 `json:"divergentKeys"` + // OrphanKeys were present on the target with no source counterpart + // (deleted when the pass deletes orphans). + OrphanKeys int64 `json:"orphanKeys"` + // Repaired is true when the pass wrote fixes rather than only reporting. + Repaired bool `json:"repaired"` +} + +// CutoverStatus tracks a Drain-mode cutover; see EtcdMirrorCutoverStatus. +type CutoverStatus struct { + // DrainTargetRevision is the source revision observed when the drain + // started — the revision the watermark must reach. + DrainTargetRevision int64 `json:"drainTargetRevision"` + // DrainedRevision is the watermark at which the drain completed. + DrainedRevision int64 `json:"drainedRevision"` + // VerifiedTime is when the post-drain verification pass succeeded. + VerifiedTime time.Time `json:"verifiedTime,omitzero"` + // SourceKeyCount / TargetKeyCount are the per-side key counts from the + // verification pass (source read pinned at DrainedRevision with an + // unpinned fallback if compacted; excluded prefixes and the reserved + // checkpoint key are not counted). + SourceKeyCount int64 `json:"sourceKeyCount"` + TargetKeyCount int64 `json:"targetKeyCount"` + // LeasedKeyCount is the lease-backed key count frozen at drain + // completion, for the runbook's purge/re-lease step. + LeasedKeyCount int64 `json:"leasedKeyCount"` +} + +// Snapshot is a point-in-time copy of the agent's state, safe to retain. +// cmd/mirror-agent's /statusz and the controller's status sync poll this +// instead of scraping internals. +// +// The JSON tags ARE the /statusz wire contract: the agent binary marshals a +// Snapshot verbatim and the controller decodes into this same type, so the +// two ends cannot drift. Zero times are omitted (omitzero = "not yet"). +type Snapshot struct { + Phase Phase `json:"phase"` + + SourceVersion string `json:"sourceVersion"` + TargetVersion string `json:"targetVersion"` + // SourceClusterID / TargetClusterID as probed at connect (0 = not yet + // probed). Both are bound into the checkpoint. JSON strings, not numbers: + // cluster IDs are random uint64s beyond 2^53, which float64-based JSON + // consumers (jq, JavaScript) would round — the same convention as the + // checkpoint and etcd's gRPC gateway. + SourceClusterID uint64 `json:"sourceClusterID,string"` + TargetClusterID uint64 `json:"targetClusterID,string"` + + // Watermark is the checkpoint watermark: the source revision through + // which the target is caught up, advanced by applies AND by watch + // progress notifications on idle prefixes. The fenced checkpoint key in + // the target etcd is the authoritative copy; this mirrors it. + Watermark int64 `json:"watermark"` + // SourceRevision is the source cluster's revision as of the last watch + // header. Cluster-global: it advances on out-of-prefix writes, so + // SourceRevision-Watermark overstates lag for prefix-scoped mirrors. + SourceRevision int64 `json:"sourceRevision"` + // LastProgressTime is when the watermark last advanced. This — not + // apply activity — is the liveness signal: an idle prefix on a live + // watch keeps progressing via notifications. + LastProgressTime time.Time `json:"lastProgressTime,omitzero"` + + InitialSyncKeyCount int64 `json:"initialSyncKeyCount"` + InitialSyncTotalKeyCount int64 `json:"initialSyncTotalKeyCount"` + InitialSyncStartTime time.Time `json:"initialSyncStartTime,omitzero"` + InitialSyncCompletionTime time.Time `json:"initialSyncCompletionTime,omitzero"` + + // KeysAppliedTotal is the monotonic count of data operations (puts plus + // deletes; the checkpoint Put rides free) committed to the target in + // fenced Txns, across scans, tails, repairs, and prunes. The denominator + // for apply-rate metrics. + KeysAppliedTotal int64 `json:"keysAppliedTotal"` + + // LeaseBackedKeyCount is the number of mirrored keys whose source copy + // is lease-backed (kv.Lease != 0). Mirrored copies are NOT lease-backed + // — leases are stripped — so a nonzero count means the cutover + // runbook's purge/re-lease step applies. + LeaseBackedKeyCount int64 `json:"leaseBackedKeyCount"` + + // ForcedResyncCount is monotonic, never reset. LastResyncReason is the + // most recent trigger. ForcedResyncCountByReason splits the same count + // by trigger (the labeled-counter surface; values sum to + // ForcedResyncCount). ResyncLoopDetected latches when + // ResyncLoopThreshold consecutive resyncs completed without reaching + // steady state (the livelock signature of retention < scan+drain time); + // it clears only when steady state is reached. + ForcedResyncCount int64 `json:"forcedResyncCount"` + ForcedResyncCountByReason map[ResyncReason]int64 `json:"forcedResyncCountByReason,omitempty"` + LastResyncReason ResyncReason `json:"lastResyncReason"` + ResyncLoopDetected bool `json:"resyncLoopDetected"` + + // ScanRestartCount is monotonic: genesis scan attempts aborted and + // restarted from a fresh R0 (see ScanRestartCause). Distinct from + // ForcedResyncCount — a restart is a bounded retry within InitialSync, + // not a checkpoint invalidation — but restarts count toward the same + // resync-loop detector. + ScanRestartCount int64 `json:"scanRestartCount"` + LastScanRestartCause ScanRestartCause `json:"lastScanRestartCause"` + + // SourceKeyCount / TargetKeyCount are the per-side in-scope key counts + // observed by the most recent reconciliation, prune, or drain + // verification pass (excluded prefixes and the reserved checkpoint key + // not counted). Populated by every pass that runs regardless of config — + // forced-resync sweeps, the OverwriteAndPrune genesis pass, and drain + // verification — plus the periodic pass when Config.ReconcileInterval + // enables it, but NOT refreshed outside those passes: a healthy + // mirror that never resyncs only gets counts from an enabled periodic + // pass. This is the equality signal the controller's InvariantsHeld + // condition reads. + SourceKeyCount int64 `json:"sourceKeyCount"` + TargetKeyCount int64 `json:"targetKeyCount"` + + // Condition-shaped flags. + Throttled bool `json:"throttled"` + QuotaExhausted bool `json:"quotaExhausted"` + Compacted bool `json:"compacted"` + + // LastReconcileTime is when the most recent reconciliation/verification + // pass completed — periodic or mandatory, including the count-only drain + // verification — the freshness input to the controller's InvariantsHeld + // condition. LastReconcileDrift is the most recent FULL diff's outcome; + // count-only verifications never overwrite it (a count check cannot + // attest DivergentKeys). + LastReconcileTime time.Time `json:"lastReconcileTime,omitzero"` + LastReconcileDrift *Drift `json:"lastReconcileDrift,omitempty"` + + // LastError / LastErrorClass describe the most recent classified + // failure ("" when the last attempt succeeded). + LastError string `json:"lastError"` + LastErrorClass Class `json:"lastErrorClass"` + // LastErrorReason is the typed-reason form of LastError ("" when untyped + // or the last attempt succeeded) — the controller's Failed-phase reason + // source, so it never string-matches error messages. + LastErrorReason string `json:"lastErrorReason"` + + // SourceLearner / TargetLearner report IsLearner from the most recent + // maintenance Status() probe of that side (the LearnerEndpoint + // condition's input). Best-effort: reflects the probed endpoint only. + SourceLearner bool `json:"sourceLearner"` + TargetLearner bool `json:"targetLearner"` + + // Cutover is populated once a drain starts; CutoverReady flips when the + // fence role is Primary. + CutoverReady bool `json:"cutoverReady"` + Cutover *CutoverStatus `json:"cutover,omitempty"` +} diff --git a/pkg/mirroragent/snapshot_test.go b/pkg/mirroragent/snapshot_test.go new file mode 100644 index 00000000..1001b243 --- /dev/null +++ b/pkg/mirroragent/snapshot_test.go @@ -0,0 +1,188 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "encoding/json" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func newTestAgent(t *testing.T) *Agent { + t.Helper() + a, err := New(Config{LinkUID: "link", Epoch: 1}, nil, nil) + require.NoError(t, err) + return a +} + +// TestApplySuccessClearsErrorFields: the wire contract for all three +// last-error fields is "" once the last attempt succeeded — a healed +// transient (e.g. a pre-claim fence race) must not leave a stale +// lastErrorReason behind. +func TestApplySuccessClearsErrorFields(t *testing.T) { + a := newTestAgent(t) + a.recordErr(&FenceError{Detail: "raced"}, ClassTransient) + s := a.Snapshot() + require.Equal(t, "FenceLost", s.LastErrorReason) + require.NotEmpty(t, s.LastError) + + a.noteApplySuccess(2, PhaseSyncing) + s = a.Snapshot() + require.Empty(t, s.LastError) + require.Empty(t, s.LastErrorClass) + require.Empty(t, s.LastErrorReason) + require.Equal(t, int64(2), s.KeysAppliedTotal) +} + +func TestSnapshotForcedResyncCountByReason(t *testing.T) { + a := newTestAgent(t) + a.noteResync(&ResyncError{Reason: ResyncReasonCompacted, Cause: errors.New("x")}) + a.noteResync(&ResyncError{Reason: ResyncReasonClusterIDMismatch, Cause: errors.New("y")}) + a.noteResync(errors.New("bare errors default to Compacted")) + + s := a.Snapshot() + require.Equal(t, int64(3), s.ForcedResyncCount) + require.Equal(t, map[ResyncReason]int64{ + ResyncReasonCompacted: 2, + ResyncReasonClusterIDMismatch: 1, + }, s.ForcedResyncCountByReason) + require.Equal(t, ResyncReasonCompacted, s.LastResyncReason) +} + +func TestSnapshotDeepCopiesReasonMap(t *testing.T) { + a := newTestAgent(t) + a.noteResync(&ResyncError{Reason: ResyncReasonCompacted, Cause: errors.New("x")}) + a.update(func(s *Snapshot) { + s.Cutover = &CutoverStatus{DrainTargetRevision: 7} + s.LastReconcileDrift = &Drift{MissingKeys: 1} + }) + + got := a.Snapshot() + got.ForcedResyncCountByReason[ResyncReasonClusterIDMismatch] = 99 + got.Cutover.DrainTargetRevision = 99 + got.LastReconcileDrift.MissingKeys = 99 + + fresh := a.Snapshot() + require.Equal(t, map[ResyncReason]int64{ResyncReasonCompacted: 1}, fresh.ForcedResyncCountByReason) + require.Equal(t, int64(7), fresh.Cutover.DrainTargetRevision) + require.Equal(t, int64(1), fresh.LastReconcileDrift.MissingKeys) +} + +// TestSnapshotJSONWireNames pins the /statusz wire contract: the exact +// lowerCamel key set, zero times absent (omitzero), nil pointers absent. +func TestSnapshotJSONWireNames(t *testing.T) { + now := time.Now() + full := Snapshot{ + Phase: PhaseSyncing, + SourceVersion: "3.5.9", + TargetVersion: "3.6.0", + SourceClusterID: 1, + TargetClusterID: 2, + Watermark: 3, + SourceRevision: 4, + LastProgressTime: now, + InitialSyncKeyCount: 5, + InitialSyncTotalKeyCount: 6, + InitialSyncStartTime: now, + InitialSyncCompletionTime: now, + KeysAppliedTotal: 7, + LeaseBackedKeyCount: 8, + ForcedResyncCount: 9, + ForcedResyncCountByReason: map[ResyncReason]int64{ResyncReasonCompacted: 9}, + LastResyncReason: ResyncReasonCompacted, + ResyncLoopDetected: true, + ScanRestartCount: 10, + LastScanRestartCause: ScanRestartWatchBufferOverflow, + SourceKeyCount: 11, + TargetKeyCount: 12, + Throttled: true, + QuotaExhausted: true, + Compacted: true, + LastReconcileTime: now, + LastReconcileDrift: &Drift{MissingKeys: 1, DivergentKeys: 2, OrphanKeys: 3, Repaired: true}, + LastError: "boom", + LastErrorClass: ClassTransient, + LastErrorReason: "InvalidConfig", + SourceLearner: true, + TargetLearner: true, + CutoverReady: true, + Cutover: &CutoverStatus{ + DrainTargetRevision: 1, DrainedRevision: 2, VerifiedTime: now, + SourceKeyCount: 3, TargetKeyCount: 4, LeasedKeyCount: 5, + }, + } + + keys := func(v any) map[string]any { + t.Helper() + raw, err := json.Marshal(v) + require.NoError(t, err) + var m map[string]any + require.NoError(t, json.Unmarshal(raw, &m)) + return m + } + + fullKeys := keys(full) + // Cluster IDs are JSON strings: as uint64s beyond 2^53 they would round + // through float64-based consumers (jq, JavaScript) as plain numbers. + require.Equal(t, "1", fullKeys["sourceClusterID"]) + require.Equal(t, "2", fullKeys["targetClusterID"]) + wantFull := []string{ + "phase", "sourceVersion", "targetVersion", "sourceClusterID", "targetClusterID", + "watermark", "sourceRevision", "lastProgressTime", + "initialSyncKeyCount", "initialSyncTotalKeyCount", "initialSyncStartTime", + "initialSyncCompletionTime", "keysAppliedTotal", "leaseBackedKeyCount", + "forcedResyncCount", "forcedResyncCountByReason", "lastResyncReason", + "resyncLoopDetected", "scanRestartCount", "lastScanRestartCause", + "sourceKeyCount", "targetKeyCount", "throttled", "quotaExhausted", "compacted", + "lastReconcileTime", "lastReconcileDrift", "lastError", "lastErrorClass", + "lastErrorReason", "sourceLearner", "targetLearner", + "cutoverReady", "cutover", + } + require.ElementsMatch(t, wantFull, mapKeys(fullKeys)) + require.ElementsMatch(t, + []string{"missingKeys", "divergentKeys", "orphanKeys", "repaired"}, + mapKeys(fullKeys["lastReconcileDrift"].(map[string]any))) + require.ElementsMatch(t, + []string{"drainTargetRevision", "drainedRevision", "verifiedTime", + "sourceKeyCount", "targetKeyCount", "leasedKeyCount"}, + mapKeys(fullKeys["cutover"].(map[string]any))) + + // Zero value: zero times, the nil map, and nil pointers are absent. + zeroKeys := keys(Snapshot{}) + for _, absent := range []string{ + "lastProgressTime", "initialSyncStartTime", "initialSyncCompletionTime", + "lastReconcileTime", "forcedResyncCountByReason", "lastReconcileDrift", "cutover", + } { + require.NotContains(t, zeroKeys, absent) + } + require.Contains(t, zeroKeys, "phase") + + // omitzero on CutoverStatus.VerifiedTime. + cutKeys := keys(CutoverStatus{}) + require.NotContains(t, cutKeys, "verifiedTime") +} + +func mapKeys(m map[string]any) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/pkg/mirroragent/tail.go b/pkg/mirroragent/tail.go new file mode 100644 index 00000000..d7a0f307 --- /dev/null +++ b/pkg/mirroragent/tail.go @@ -0,0 +1,310 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "errors" + "fmt" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +// tail is the live watch-replay loop. wch may carry a watch opened before a +// genesis scan (whose buffered events replay over the scanned base), with +// wcancel its cancel func; when wch is nil, a watch opens at fromRev+1. +// Transient watch failures re-watch from the checkpoint watermark; +// compaction of the resume revision propagates as a forced resync. The +// current watch's cancel is published via Agent.watchCancel so a stalled +// apply can stop the source stream (bounded memory during target stalls). +func (a *Agent) tail(ctx context.Context, wch clientv3.WatchChan, wcancel context.CancelFunc, fromRev int64) error { + a.setPhase(PhaseSyncing) + // Arm the periodic reconciliation deadline on first reaching steady + // state: one full interval out, never during or before a genesis scan. + if a.cfg.ReconcileInterval > 0 && a.nextReconcile.IsZero() { + a.scheduleNextReconcile() + } + srcStart, srcEnd := a.rw.sourceRange() + for { + if wch == nil { + rev := a.watermark() + if rev < fromRev { + rev = fromRev + } + var wctx context.Context + wctx, wcancel = context.WithCancel(clientv3.WithRequireLeader(ctx)) + wch = a.src.Watch(wctx, srcStart, clientv3.WithRange(srcEnd), + clientv3.WithRev(rev+1), clientv3.WithProgressNotify()) + } + a.watchCancel = wcancel + err := a.consume(ctx, wch) + a.watchCancel = nil + if wcancel != nil { + wcancel() + } + wch, wcancel = nil, nil + if errors.Is(err, errDrained) || ctx.Err() != nil { + return err + } + switch class := Classify(err); class { + case ClassTransient, ClassThrottle: + a.recordErr(err, class) + a.setPhase(PhaseDegraded) + if serr := sleepCtx(ctx, a.bo.next(class)); serr != nil { + return serr + } + a.setPhase(PhaseSyncing) + default: + return err + } + } +} + +// consume applies watch responses until the channel closes or fails. It +// drives client-side progress requests (server-side notify intervals are +// uncontrollable on foreign clusters) and checks the drain gate between +// responses. +func (a *Agent) consume(ctx context.Context, wch clientv3.WatchChan) error { + ticker := time.NewTicker(a.cfg.ProgressInterval) + defer ticker.Stop() + // The periodic reconciliation deadline rides the same select (nil channel + // when disabled, so the case never fires). The timer is rebuilt from the + // persistent deadline on every consume entry — a mandatory sweep between + // cycles re-armed it and is picked up automatically. + var reconcileC <-chan time.Time + var reconcileTimer *time.Timer + if a.cfg.ReconcileInterval > 0 { + reconcileTimer = time.NewTimer(time.Until(a.nextReconcile)) + defer reconcileTimer.Stop() + reconcileC = reconcileTimer.C + } + for { + if err := a.maybeDrain(ctx); err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-reconcileC: + if err := a.maybeReconcile(ctx); err != nil { + return err + } + reconcileTimer.Reset(time.Until(a.nextReconcile)) + case <-ticker.C: + // The progress request MUST carry the same outgoing metadata as + // the Watch call: clientv3 keys watcher gRPC streams by ctx + // metadata, and every engine watch is opened WithRequireLeader. + // A bare ctx here would address an empty stream and no + // notification would ever arrive on an idle prefix. + rctx, cancel := context.WithTimeout(clientv3.WithRequireLeader(ctx), a.cfg.RequestTimeout) + _ = a.src.RequestProgress(rctx) + cancel() + case wr, ok := <-wch: + if !ok { + return fmt.Errorf("source watch channel closed") + } + if wr.CompactRevision != 0 { + return &ResyncError{Reason: ResyncReasonCompacted, Cause: wr.Err()} + } + if err := wr.Err(); err != nil { + return err + } + if err := a.handleResponse(ctx, &wr, true); err != nil { + return err + } + } + } +} + +// handleResponse applies one watch response: whole revisions coalesced into +// fenced Txns, flushed only at source-revision boundaries; trusted progress +// notifications advance the watermark checkpoint on idle prefixes. live is +// true only for responses consumed from a live tail channel — genesis +// replay-buffer applies pass false so they never reset the resync-loop +// detector from inside the very resync it is counting. +func (a *Agent) handleResponse(ctx context.Context, wr *clientv3.WatchResponse, live bool) error { + hdrRev := wr.Header.Revision + a.update(func(s *Snapshot) { + if hdrRev > s.SourceRevision { + s.SourceRevision = hdrRev + } + }) + if wr.IsProgressNotify() { + if !a.trustProgress || hdrRev <= a.watermark() { + return nil + } + if err := a.applyFenced(ctx, nil, a.newFence(hdrRev, false, "", 0), "", 0); err != nil { + return err + } + a.advanceWatermark(hdrRev) + return nil + } + + ops := make([]kvOp, 0, len(wr.Events)) + revs := make([]int64, 0, len(wr.Events)) + var leased int64 + for _, ev := range wr.Events { + srcKey := string(ev.Kv.Key) + dstKey, ok := a.rw.rewrite(srcKey) + if !ok { + continue + } + op := kvOp{key: dstKey, srcKey: srcKey} + if ev.Type == clientv3.EventTypeDelete { + op.isDelete = true + } else { + op.value = string(ev.Kv.Value) + if ev.Kv.Lease != 0 { + leased++ + } + } + ops = append(ops, op) + revs = append(revs, ev.Kv.ModRevision) + } + if leased > 0 { + a.update(func(s *Snapshot) { s.LeaseBackedKeyCount += leased }) + } + if len(ops) == 0 { + return nil + } + // A source revision's events never span watch responses, so flushing at + // the end of the response only ever cuts at a revision boundary. + b := newBatcher(a.cfg.MaxTxnOps, a.cfg.TxnFlushBytes) + for _, g := range groupByRevision(ops, revs) { + for _, fs := range b.add(g) { + if err := a.applyLiveFlush(ctx, &fs); err != nil { + return err + } + } + } + if fs := b.flush(); fs != nil { + if err := a.applyLiveFlush(ctx, fs); err != nil { + return err + } + } + if live { + a.steadyState() + } + return nil +} + +// applyLiveFlush applies one revision-complete flush set; the checkpoint +// watermark advances to the set's last complete revision in the same Txn. +func (a *Agent) applyLiveFlush(ctx context.Context, fs *flushSet) error { + if err := a.applyOps(ctx, fs, a.newFence(fs.watermark, false, "", 0)); err != nil { + return err + } + a.advanceWatermark(fs.watermark) + return nil +} + +// maybeDrain drives Drain mode: record the drain target revision once, then +// complete the cutover when the checkpoint watermark reaches it. +func (a *Agent) maybeDrain(ctx context.Context) error { + if a.cfg.Mode != ModeDrain && !a.drainReq.Load() { + return nil + } + snap := a.Snapshot() + if snap.Cutover == nil { + srcStart, srcEnd := a.rw.sourceRange() + var target int64 + if a.trustProgress { + resp, err := a.getRetry(ctx, a.src, srcStart, + clientv3.WithRange(srcEnd), clientv3.WithCountOnly()) + if err != nil { + return err + } + target = resp.Header.Revision + } else { + // Below the progress-trust floor (source < 3.4.25 / 3.5.8) the + // watermark advances ONLY on applied in-prefix events, so a + // cluster-revision drain target would never terminate on a shared + // source: out-of-prefix writes push it past the last in-prefix + // event while the drain itself quiesces in-prefix writers. Fall + // back to the highest in-scope mod revision; a tombstone above it + // (an in-flight delete) is caught and repaired by the drain + // verification pass before the role flips. + resp, err := a.getRetry(ctx, a.src, srcStart, clientv3.WithRange(srcEnd), + clientv3.WithSort(clientv3.SortByModRevision, clientv3.SortDescend), + clientv3.WithLimit(1), clientv3.WithKeysOnly()) + if err != nil { + return err + } + if len(resp.Kvs) > 0 { + target = resp.Kvs[0].ModRevision + } else { + target = a.watermark() + } + } + a.update(func(s *Snapshot) { + s.Cutover = &CutoverStatus{DrainTargetRevision: target} + if target > s.SourceRevision { + s.SourceRevision = target + } + }) + snap = a.Snapshot() + } + if snap.Watermark < snap.Cutover.DrainTargetRevision { + return nil + } + return a.completeDrain(ctx) +} + +// completeDrain verifies convergence, records the cutover block, and flips +// the fence role to Primary so any straggler mirror apply fails its compare +// loudly. Returns errDrained on success. +func (a *Agent) completeDrain(ctx context.Context) error { + srcN, dstN, err := a.verifyCounts(ctx) + if err != nil { + return err + } + a.recordKeyCounts(srcN, dstN) + if srcN != dstN { + // One repair+prune pass and a recount; a persisting mismatch is real + // divergence and must fail the drain rather than cut over. + if rerr := a.reconcilePass(ctx, true, true, true); rerr != nil { + return rerr + } + if srcN, dstN, err = a.verifyCounts(ctx); err != nil { + return err + } + a.recordKeyCounts(srcN, dstN) + if srcN != dstN { + return &DrainVerificationError{SourceKeys: srcN, TargetKeys: dstN} + } + } + wm := a.watermark() + f := a.newFence(wm, false, "", 0) + f.Role = RolePrimary + if err := a.applyFenced(ctx, nil, f, "", 0); err != nil { + return err + } + now := time.Now() + a.update(func(s *Snapshot) { + c := *s.Cutover + c.DrainedRevision = wm + c.VerifiedTime = now + c.SourceKeyCount = srcN + c.TargetKeyCount = dstN + c.LeasedKeyCount = s.LeaseBackedKeyCount + s.Cutover = &c + s.CutoverReady = true + s.Phase = PhaseDrained + }) + return errDrained +} diff --git a/pkg/mirroragent/version.go b/pkg/mirroragent/version.go new file mode 100644 index 00000000..87cea9f2 --- /dev/null +++ b/pkg/mirroragent/version.go @@ -0,0 +1,65 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "fmt" + + "github.com/coreos/go-semver/semver" +) + +// hardVersionFloor is the declared minimum etcd server version. Below it the +// agent fails permanently (UnsupportedVersion) rather than degrading in +// undefined ways. +const hardVersionFloor = "3.4.0" + +// Watch progress notifications are unreliable below 3.4.25 / 3.5.8. Below +// these floors the agent does not trust progress notifications: the +// watermark only advances on applies, so idle prefixes resync from scratch +// after any restart longer than retention, and a Drain may not terminate on +// a quiet prefix. +var progressTrustFloors = map[int64]semver.Version{ + 4: {Major: 3, Minor: 4, Patch: 25}, + 5: {Major: 3, Minor: 5, Patch: 8}, +} + +// versionInfo is the outcome of the connect-time maintenance Status() probe. +type versionInfo struct { + Version string + // TrustProgressNotify gates the watermark machinery that drives lag, + // idle-prefix checkpointing, and the Drain gate. + TrustProgressNotify bool +} + +// classifyVersion enforces the hard floor and derives progress trust. +// side is "source" or "target"; the hard floor is enforced on both, the +// progress-trust floor only matters for the source (the watched side). +func classifyVersion(side, version string) (versionInfo, error) { + v, err := semver.NewVersion(version) + if err != nil { + return versionInfo{}, fmt.Errorf("unparseable %s etcd version %q: %w", side, version, err) + } + floor := semver.New(hardVersionFloor) + if v.LessThan(*floor) { + return versionInfo{}, &UnsupportedVersionError{Side: side, Version: version} + } + info := versionInfo{Version: version, TrustProgressNotify: true} + if trustFloor, ok := progressTrustFloors[v.Minor]; ok && v.Major == 3 { + info.TrustProgressNotify = !v.LessThan(trustFloor) + } + return info, nil +} diff --git a/pkg/mirroragent/version_test.go b/pkg/mirroragent/version_test.go new file mode 100644 index 00000000..2a29a1b9 --- /dev/null +++ b/pkg/mirroragent/version_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestClassifyVersion pins the version gates: the >=3.4.0 hard floor (below +// it the agent fails permanently) and the 3.4.25/3.5.8 progress-trust floor +// (below it progress notifications can report revisions ahead of delivered +// events, so trusting them would checkpoint the watermark past undelivered +// data — silent loss after a restart). +func TestClassifyVersion(t *testing.T) { + cases := []struct { + version string + wantErr bool + wantFloor bool // UnsupportedVersionError specifically + wantTrust bool + }{ + {version: "3.3.9", wantErr: true, wantFloor: true}, + {version: "3.0.0", wantErr: true, wantFloor: true}, + {version: "3.4.0", wantTrust: false}, + {version: "3.4.24", wantTrust: false}, + {version: "3.4.25", wantTrust: true}, + {version: "3.4.33", wantTrust: true}, + {version: "3.5.0", wantTrust: false}, + {version: "3.5.7", wantTrust: false}, + {version: "3.5.8", wantTrust: true}, + {version: "3.6.0", wantTrust: true}, + {version: "3.6.12", wantTrust: true}, + {version: "4.0.0", wantTrust: true}, + {version: "garbage", wantErr: true}, + {version: "", wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.version, func(t *testing.T) { + info, err := classifyVersion("source", tc.version) + if tc.wantErr { + require.Error(t, err) + var uv *UnsupportedVersionError + if tc.wantFloor { + require.ErrorAs(t, err, &uv) + assert.Equal(t, "source", uv.Side) + assert.Equal(t, ClassPermanent, Classify(err)) + } + return + } + require.NoError(t, err) + assert.Equal(t, tc.version, info.Version) + assert.Equal(t, tc.wantTrust, info.TrustProgressNotify, + "progress-trust gate for %s", tc.version) + }) + } +} diff --git a/test/e2e/mirror_agent_test.go b/test/e2e/mirror_agent_test.go new file mode 100644 index 00000000..436b94c4 --- /dev/null +++ b/test/e2e/mirror_agent_test.go @@ -0,0 +1,469 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/e2e-framework/klient/k8s" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/envfuncs" + "sigs.k8s.io/e2e-framework/pkg/features" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + "go.etcd.io/etcd-operator/pkg/mirroragent" +) + +const ( + mirrorNamespace = "mirror-e2e" + mirrorSourceName = "mirror-source" + mirrorTargetName = "mirror-target" + mirrorAgentName = "mirror-agent" + mirrorAgentPort = 8080 + mirrorKeyPrefix = "/mirror-e2e/" + mirrorKeyCount = 10 + // replicationWait bounds each replication poll loop. Replication itself + // is sub-second; the ceiling absorbs API-server exec round-trip latency + // on loaded CI, and the logged elapsed time carries the fast-path proof. + replicationWait = 10 * time.Second + replicationPoll = 250 * time.Millisecond +) + +// createMirrorEtcdCluster creates a size-1 cleartext EtcdCluster in +// mirrorNamespace. +func createMirrorEtcdCluster(ctx context.Context, t *testing.T, cfg *envconf.Config, name string) { + t.Helper() + ec := &ecv1alpha1.EtcdCluster{ + TypeMeta: metav1.TypeMeta{APIVersion: "operator.etcd.io/v1alpha1", Kind: "EtcdCluster"}, + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: mirrorNamespace}, + Spec: ecv1alpha1.EtcdClusterSpec{Size: 1, Version: etcdVersion}, + } + if err := cfg.Client().Resources().Create(ctx, ec); err != nil { + t.Fatalf("failed to create EtcdCluster %s/%s: %v", mirrorNamespace, name, err) + } +} + +// waitForMirrorSTSReady waits for the cluster's StatefulSet in +// mirrorNamespace to report one ready replica. +func waitForMirrorSTSReady(ctx context.Context, t *testing.T, cfg *envconf.Config, name string) { + t.Helper() + client := cfg.Client() + sts := appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: mirrorNamespace}} + // Not utils.GetKubernetesResource: its poll aborts on the first NotFound + // instead of waiting out the timeout. ResourceMatch swallows Get errors. + if err := wait.For( + conditions.New(client.Resources()).ResourceMatch(&sts, func(k8s.Object) bool { return true }), + wait.WithContext(ctx), + wait.WithTimeout(3*time.Minute), + wait.WithInterval(5*time.Second), + ); err != nil { + t.Fatalf("StatefulSet %s never appeared: %v", name, err) + } + if err := wait.For( + conditions.New(client.Resources()).ResourceScaled(&sts, func(k8s.Object) int32 { + return sts.Status.ReadyReplicas + }, 1), + wait.WithContext(ctx), + wait.WithTimeout(3*time.Minute), + wait.WithInterval(5*time.Second), + ); err != nil { + t.Fatalf("StatefulSet %s never reached 1 ready replica: %v", name, err) + } +} + +// execInPodRetried retries transient exec failures (the SPDY stream setup is +// a known flake mode under CI load). Only for idempotent commands. +func execInPodRetried(t *testing.T, cfg *envconf.Config, podName string, command []string) (string, error) { + t.Helper() + var stderr string + var err error + for attempt := 0; attempt < 3; attempt++ { + if attempt > 0 { + time.Sleep(250 * time.Millisecond) + } + if _, stderr, err = execInPod(t, cfg, podName, mirrorNamespace, command); err == nil { + return "", nil + } + } + return stderr, err +} + +// deleteMirrorResources best-effort deletes everything the feature creates. +// Shared by the feature Teardown and a Setup t.Cleanup: a t.Fatalf in Setup +// runtime.Goexits past the feature teardowns, and the leaked namespace would +// wedge every ETCD_E2E_SKIP_TEARDOWN rerun on AlreadyExists. +func deleteMirrorResources(ctx context.Context, t *testing.T, cfg *envconf.Config) { + t.Helper() + client := cfg.Client() + var pod corev1.Pod + if err := client.Resources().Get(ctx, mirrorAgentName, mirrorNamespace, &pod); err == nil { + if err := client.Resources().Delete(ctx, &pod); err != nil { + t.Logf("failed to delete mirror-agent pod: %v", err) + } + } + for _, name := range []string{mirrorSourceName, mirrorTargetName} { + var ec ecv1alpha1.EtcdCluster + if err := client.Resources().Get(ctx, name, mirrorNamespace, &ec); err == nil { + if err := client.Resources().Delete(ctx, &ec); err != nil { + t.Logf("failed to delete EtcdCluster %s: %v", name, err) + } + } + } + var ns corev1.Namespace + if err := client.Resources().Get(ctx, mirrorNamespace, "", &ns); err == nil { + if err := client.Resources().Delete(ctx, &ns); err != nil { + t.Logf("failed to delete namespace %s: %v", mirrorNamespace, err) + } + } +} + +// etcdClientEndpoint is the per-pod DNS client URL the operator itself +// advertises for member 0 of a size-1 cluster. +func etcdClientEndpoint(name string) string { + return fmt.Sprintf("http://%s-0.%s.%s.svc.cluster.local:2379", name, name, mirrorNamespace) +} + +// mirrorAgentPod renders the agent pod wired source→target, cleartext, with +// the observability listener exposed on mirrorAgentPort. +func mirrorAgentPod() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: mirrorAgentName, + Namespace: mirrorNamespace, + Labels: map[string]string{"app": mirrorAgentName}, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, // a startup failure must surface, not crash-loop + Containers: []corev1.Container{{ + Name: mirrorAgentName, + Image: imageName, + ImagePullPolicy: corev1.PullIfNotPresent, // kind-loaded, never pulled + // The image ENTRYPOINT is /manager; the same image ships the agent. + Command: []string{"/mirror-agent"}, + Args: []string{ + "--link-uid=mirror-e2e-link", + "--epoch=1", + "--source-endpoints=" + etcdClientEndpoint(mirrorSourceName), + "--target-endpoints=" + etcdClientEndpoint(mirrorTargetName), + "--source-prefix=" + mirrorKeyPrefix, + "--target-prefix=" + mirrorKeyPrefix, + "--http-bind-address=:8080", + }, + Ports: []corev1.ContainerPort{{Name: "http", ContainerPort: mirrorAgentPort}}, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", Port: intstr.FromInt32(mirrorAgentPort), + }}, + PeriodSeconds: 2, + FailureThreshold: 60, + }, + }}, + }, + } +} + +// getViaPodProxy GETs a path on the agent pod's HTTP port through the +// API-server pod proxy (the pod image is distroless — nothing can be exec'd). +func getViaPodProxy(ctx context.Context, cfg *envconf.Config, path string) ([]byte, error) { + client := kubernetes.NewForConfigOrDie(cfg.Client().RESTConfig()) + return client.CoreV1().RESTClient().Get(). + Namespace(mirrorNamespace). + Resource("pods"). + SubResource("proxy"). + Name(fmt.Sprintf("%s:%d", mirrorAgentName, mirrorAgentPort)). + Suffix(path). + Do(ctx).Raw() +} + +// getStatusz decodes /statusz into mirroragent.Snapshot — the JSON tags on +// that type are the wire contract, so decoding into it cannot drift. +func getStatusz(ctx context.Context, cfg *envconf.Config) (mirroragent.Snapshot, error) { + raw, err := getViaPodProxy(ctx, cfg, "statusz") + if err != nil { + return mirroragent.Snapshot{}, err + } + var s mirroragent.Snapshot + if err := json.Unmarshal(raw, &s); err != nil { + return mirroragent.Snapshot{}, fmt.Errorf("decoding /statusz %q: %w", raw, err) + } + return s, nil +} + +// scrapeKeysAppliedMetric returns the unlabeled +// etcd_mirror_agent_keys_applied_total sample from /metrics. Parsed by hand +// to avoid promoting prometheus/common to a direct dependency. +func scrapeKeysAppliedMetric(ctx context.Context, t *testing.T, cfg *envconf.Config) float64 { + t.Helper() + raw, err := getViaPodProxy(ctx, cfg, "metrics") + if err != nil { + t.Fatalf("failed to scrape /metrics: %v", err) + } + for _, line := range strings.Split(string(raw), "\n") { + if strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + if len(fields) == 2 && fields[0] == "etcd_mirror_agent_keys_applied_total" { + v, err := strconv.ParseFloat(fields[1], 64) + if err != nil { + t.Fatalf("unparsable keys_applied_total sample %q: %v", line, err) + } + return v + } + } + t.Fatalf("etcd_mirror_agent_keys_applied_total absent from /metrics:\n%s", raw) + return 0 +} + +// dumpAgentDiagnostics logs the raw /statusz body and the agent pod logs so +// a failed poll is debuggable from CI output alone. +func dumpAgentDiagnostics(ctx context.Context, t *testing.T, cfg *envconf.Config) { + t.Helper() + if raw, err := getViaPodProxy(ctx, cfg, "statusz"); err != nil { + t.Logf("statusz unavailable: %v", err) + } else { + t.Logf("last /statusz: %s", raw) + } + client := kubernetes.NewForConfigOrDie(cfg.Client().RESTConfig()) + logs, err := client.CoreV1().Pods(mirrorNamespace). + GetLogs(mirrorAgentName, &corev1.PodLogOptions{}).Do(ctx).Raw() + if err != nil { + t.Logf("agent pod logs unavailable: %v", err) + return + } + t.Logf("mirror-agent pod logs:\n%s", logs) +} + +// hasAllMirrorKeys reports whether plain etcdctl get output (alternating +// key and value lines) contains every expected key with its expected value. +func hasAllMirrorKeys(stdout string) bool { + lines := strings.Split(strings.TrimSpace(stdout), "\n") + got := make(map[string]string, len(lines)/2) + for i := 0; i+1 < len(lines); i += 2 { + got[lines[i]] = lines[i+1] + } + for i := range mirrorKeyCount { + if got[fmt.Sprintf("%skey-%02d", mirrorKeyPrefix, i)] != fmt.Sprintf("val-%02d", i) { + return false + } + } + return true +} + +// TestMirrorAgent smoke-tests the mirror-agent binary: two size-1 etcd +// clusters, an agent pod wired source→target, put/delete replication within +// a polled ceiling, and advancing /statusz and /metrics counters. +func TestMirrorAgent(t *testing.T) { + feature := features.New("mirror-agent") + + var baseline mirroragent.Snapshot + var writeTime time.Time + + feature.Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + client := cfg.Client() + + // A prior failed or just-finished run can leave the namespace behind + // (or still Terminating); purge it so reruns self-heal. + var leftover corev1.Namespace + if err := client.Resources().Get(ctx, mirrorNamespace, "", &leftover); err == nil { + t.Logf("deleting leftover namespace %s from a previous run", mirrorNamespace) + _ = client.Resources().Delete(ctx, &leftover) + if err := wait.For(func(ctx context.Context) (bool, error) { + var ns corev1.Namespace + return apierrors.IsNotFound(client.Resources().Get(ctx, mirrorNamespace, "", &ns)), nil + }, wait.WithTimeout(2*time.Minute), wait.WithInterval(2*time.Second)); err != nil { + t.Fatalf("leftover namespace %s never finished deleting: %v", mirrorNamespace, err) + } + } + + ctx, err := envfuncs.CreateNamespace(mirrorNamespace)(ctx, cfg) + if err != nil { + t.Fatalf("failed to create namespace %s: %v", mirrorNamespace, err) + } + // Unlike the feature Teardown, t.Cleanup still runs after a Setup + // t.Fatalf; deleteMirrorResources is idempotent, so double cleanup + // on the happy path is harmless. + t.Cleanup(func() { + deleteMirrorResources(context.Background(), t, cfg) + }) + + createMirrorEtcdCluster(ctx, t, cfg, mirrorSourceName) + createMirrorEtcdCluster(ctx, t, cfg, mirrorTargetName) + waitForMirrorSTSReady(ctx, t, cfg, mirrorSourceName) + waitForMirrorSTSReady(ctx, t, cfg, mirrorTargetName) + + // Both sides are up, so the agent never lingers in Connecting. + pod := mirrorAgentPod() + if err := client.Resources().Create(ctx, pod); err != nil { + t.Fatalf("failed to create mirror-agent pod: %v", err) + } + // Not conditions.PodReady: that spins the full timeout on a pod that + // already crashed (RestartPolicy=Never) and aborts on transient Get + // errors. Fail fast on a terminal phase, swallow Get blips. + if err := wait.For(func(ctx context.Context) (bool, error) { + var p corev1.Pod + if gerr := client.Resources().Get(ctx, mirrorAgentName, mirrorNamespace, &p); gerr != nil { + return false, nil //nolint:nilerr // transient API error; keep polling + } + if p.Status.Phase == corev1.PodFailed || p.Status.Phase == corev1.PodSucceeded { + return false, fmt.Errorf("pod is terminal (%s)", p.Status.Phase) + } + for _, cond := range p.Status.Conditions { + if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue { + return true, nil + } + } + return false, nil + }, wait.WithTimeout(3*time.Minute), wait.WithInterval(2*time.Second)); err != nil { + dumpAgentDiagnostics(ctx, t, cfg) + t.Fatalf("mirror-agent pod never became ready: %v", err) + } + return ctx + }) + + feature.Assess("agent reaches Syncing (baseline)", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + // readyz turns ready at InitialSync already, so gate on the + // statusz phase rather than pod readiness alone. + if err := wait.For(func(ctx context.Context) (bool, error) { + s, err := getStatusz(ctx, cfg) + if err != nil { + return false, nil //nolint:nilerr // proxy may 503 while the pod settles; keep polling + } + baseline = s + return s.Phase == mirroragent.PhaseSyncing, nil + }, wait.WithTimeout(2*time.Minute), wait.WithInterval(time.Second)); err != nil { + dumpAgentDiagnostics(ctx, t, cfg) + t.Fatalf("agent never reached phase %s (last snapshot: %+v): %v", + mirroragent.PhaseSyncing, baseline, err) + } + if baseline.SourceClusterID == baseline.TargetClusterID { + t.Fatalf("source and target cluster IDs both %d: agent is not wired to two distinct clusters", + baseline.SourceClusterID) + } + return ctx + }) + + feature.Assess("writes replicate source to target", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + sourcePod := mirrorSourceName + "-0" + for i := range mirrorKeyCount { + key := fmt.Sprintf("%skey-%02d", mirrorKeyPrefix, i) + val := fmt.Sprintf("val-%02d", i) + if stderr, err := execInPodRetried(t, cfg, sourcePod, + []string{"etcdctl", "put", key, val}); err != nil { + t.Fatalf("failed to put %s on source: %v, stderr: %s", key, err, stderr) + } + } + writeTime = time.Now() + + // Range over key- specifically: a bare prefix get would also + // return the agent's reserved checkpoint key. + targetPod := mirrorTargetName + "-0" + if err := wait.For(func(context.Context) (bool, error) { + stdout, _, err := execInPod(t, cfg, targetPod, mirrorNamespace, + []string{"etcdctl", "get", "--prefix", mirrorKeyPrefix + "key-"}) + if err != nil { + return false, nil //nolint:nilerr // transient exec failure; keep polling + } + return hasAllMirrorKeys(stdout), nil + }, wait.WithTimeout(replicationWait), wait.WithInterval(replicationPoll)); err != nil { + dumpAgentDiagnostics(ctx, t, cfg) + t.Fatalf("target never showed all %d keys within %s: %v", mirrorKeyCount, replicationWait, err) + } + t.Logf("all %d keys visible on target after %s", mirrorKeyCount, time.Since(writeTime)) + return ctx + }) + + feature.Assess("delete replicates", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + key := mirrorKeyPrefix + "key-00" + if stderr, err := execInPodRetried(t, cfg, mirrorSourceName+"-0", + []string{"etcdctl", "del", key}); err != nil { + t.Fatalf("failed to delete %s on source: %v, stderr: %s", key, err, stderr) + } + delTime := time.Now() + + if err := wait.For(func(context.Context) (bool, error) { + stdout, _, err := execInPod(t, cfg, mirrorTargetName+"-0", mirrorNamespace, + []string{"etcdctl", "get", key, "--print-value-only"}) + if err != nil { + return false, nil //nolint:nilerr // transient exec failure; keep polling + } + return strings.TrimSpace(stdout) == "", nil + }, wait.WithTimeout(replicationWait), wait.WithInterval(replicationPoll)); err != nil { + dumpAgentDiagnostics(ctx, t, cfg) + t.Fatalf("delete of %s never reached the target within %s: %v", key, replicationWait, err) + } + t.Logf("delete visible on target after %s", time.Since(delTime)) + return ctx + }) + + feature.Assess("statusz and metrics advanced", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + // Poll rather than one-shot: the snapshot counters are not + // updated atomically with the txn our read-back observed. + var last mirroragent.Snapshot + if err := wait.For(func(ctx context.Context) (bool, error) { + s, err := getStatusz(ctx, cfg) + if err != nil { + return false, nil //nolint:nilerr // transient proxy failure; keep polling + } + last = s + return s.KeysAppliedTotal >= baseline.KeysAppliedTotal+mirrorKeyCount+1 && + s.Watermark > baseline.Watermark && + !s.LastProgressTime.IsZero() && + s.LastProgressTime.After(baseline.LastProgressTime) && + s.Phase == mirroragent.PhaseSyncing, nil + }, wait.WithTimeout(replicationWait), wait.WithInterval(replicationPoll)); err != nil { + dumpAgentDiagnostics(ctx, t, cfg) + t.Fatalf("statusz never advanced past baseline\nbaseline: %+v\nlast: %+v\nerror: %v", + baseline, last, err) + } + // Transient errors are normal operation (LastError only clears + // on the next successful commit); only a permanent one is a bug. + if last.LastErrorClass == mirroragent.ClassPermanent { + t.Errorf("agent reports permanent error %q", last.LastError) + } + // Scraped after statusz and monotonic, so it can never be less. + if v := scrapeKeysAppliedMetric(ctx, t, cfg); v < float64(last.KeysAppliedTotal) { + t.Errorf("metrics keys_applied_total %v below statusz keysAppliedTotal %d", v, last.KeysAppliedTotal) + } + return ctx + }) + + feature.Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + deleteMirrorResources(ctx, t, cfg) + return ctx + }) + + _ = testEnv.Test(t, feature.Feature()) +}