diff --git a/Makefile b/Makefile index c89d4a936..89a511a57 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,8 @@ OS := $(shell go env GOOS) KUBE_MAJOR_VERSION := 1 KUBE_MINOR_VERSION := $(shell go mod edit -json | jq '.Require[] | select(.Path == "k8s.io/client-go") | .Version' --raw-output | sed "s/v[0-9]*\.\([0-9]*\).*/\1/") GIT_COMMIT := $(shell git rev-parse --short HEAD || echo 'local') -GIT_DIRTY := $(shell git diff --quiet && echo 'clean' || echo 'dirty') +# --quiet would still produces output when files are deleted +GIT_DIRTY := $(shell git diff --quiet >/dev/null && echo 'clean' || echo 'dirty') GIT_VERSION := $(shell go mod edit -json | jq '.Require[] | select(.Path == "k8s.io/client-go") | .Version' --raw-output | sed 's/v0/v1/')+kube-bind-$(shell git describe --tags --match='v*' --abbrev=14 "$(GIT_COMMIT)^{commit}" 2>/dev/null || echo v0.0.0-$(GIT_COMMIT)) BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') LDFLAGS := \ diff --git a/README.md b/README.md index 7075a7fa3..105ba4ab4 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ $ kubectl krew index add bind https://github.com/kube-bind/krew-index.git $ kubectl krew install bind/bind $ kubectl bind login https://mangodb $ kubectl bind -Redirect to the brower to authenticate via OIDC. +Redirect to the browser to authenticate via OIDC. BOOM – the MangoDB API is available in the local cluster, without anything MangoDB-specific running. $ kubectl get mangodbs diff --git a/backend/controllers/serviceexportrequest/serviceexportrequest_reconcile.go b/backend/controllers/serviceexportrequest/serviceexportrequest_reconcile.go index 59c3584ef..42533b75f 100644 --- a/backend/controllers/serviceexportrequest/serviceexportrequest_reconcile.go +++ b/backend/controllers/serviceexportrequest/serviceexportrequest_reconcile.go @@ -163,6 +163,14 @@ func (r *reconciler) ensureBoundSchemas(ctx context.Context, cl client.Client, c continue } + // If namespaced isolation is configured for cluster-scoped objects, + // we need to rewrite the BoundSchema's scope accordingly. For all + // other isolation strategies, as well as for namespaced schemas, + // no changes are necessary. + if boundSchema.Spec.Scope == apiextensionsv1.NamespaceScoped && r.clusterScopedIsolation == kubebindv1alpha2.IsolationNamespaced { + boundSchema.Spec.Scope = apiextensionsv1.ClusterScoped + } + if err := r.createBoundSchema(ctx, cl, boundSchema); err != nil { return err } diff --git a/backend/oidc/oidc.go b/backend/oidc/oidc.go index 8d20680fd..a617b6047 100644 --- a/backend/oidc/oidc.go +++ b/backend/oidc/oidc.go @@ -91,7 +91,7 @@ func (s *Server) Config(callbackURL, issuerURL string) (*Config, error) { c := &Config{ ClientID: s.server.Config().ClientID, ClientSecret: s.server.Config().ClientSecret, - Issuer: issuerURL, // This overrided default fake OIDC issuer URL. Must match what it is served at. + Issuer: issuerURL, // This overrides default fake OIDC issuer URL. Must match what it is served at. AccessTTL: s.server.Config().AccessTTL, RefreshTTL: s.server.Config().RefreshTTL, diff --git a/cli/cmd/kubectl-bind/cmd/kubectlBind_test.go b/cli/cmd/kubectl-bind/cmd/kubectlBind_test.go index 7c349235d..c937e46c3 100644 --- a/cli/cmd/kubectl-bind/cmd/kubectlBind_test.go +++ b/cli/cmd/kubectl-bind/cmd/kubectlBind_test.go @@ -30,7 +30,7 @@ func TestKubectlBindCommand(t *testing.T) { require.Equal(t, "kubectl-bind", rootCmd.Use, "Unexpected one-line command description") require.Equal(t, "kubectl plugin for kube-bind, bind different remote types into the current cluster.", rootCmd.Short, "Unexpected short command description") - require.Contains(t, rootCmd.Long, "To bind a remote service, use the 'kubectl bind' command.", "Unexpected lond command Long") + require.Contains(t, rootCmd.Long, "To bind a remote service, use the 'kubectl bind' command.", "Unexpected long command") require.Equal(t, rootCmd.Example, fmt.Sprintf(bindcmd.BindExampleUses, "kubectl"), "Unexpected command Example") } diff --git a/cli/pkg/kubectl/bind-login/plugin/login.go b/cli/pkg/kubectl/bind-login/plugin/login.go index 289b14ccc..b3b898e8e 100644 --- a/cli/pkg/kubectl/bind-login/plugin/login.go +++ b/cli/pkg/kubectl/bind-login/plugin/login.go @@ -58,7 +58,7 @@ type LoginOptions struct { } // TokenResponse represents the response from the OAuth callback -// Important: this stuct must match one on backend/auth/types.go +// Important: this struct must match one on backend/auth/types.go type TokenResponse struct { // OAuth2 token fields AccessToken string `json:"access_token"` diff --git a/contrib/kcp/go.mod b/contrib/kcp/go.mod index d0dc5071f..815f165c5 100644 --- a/contrib/kcp/go.mod +++ b/contrib/kcp/go.mod @@ -14,7 +14,7 @@ replace ( // Can use versioned when v0.28.2 releases replace github.com/kcp-dev/kcp/sdk => github.com/kcp-dev/kcp/sdk v0.28.1-0.20251003164010-742ce0ea6b8c -// k/k 1.34 is leaking from main repo. This pins some deps to force depdendency tree to be on 1.34 +// k/k 1.34 is leaking from main repo. This pins some deps to force dependency tree to be on 1.34 replace ( github.com/google/gnostic-models => github.com/google/gnostic-models v0.6.9 k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff diff --git a/contrib/kcp/test/e2e/kcp_test.go b/contrib/kcp/test/e2e/kcp_test.go index 691695bcd..5ed7c74c9 100644 --- a/contrib/kcp/test/e2e/kcp_test.go +++ b/contrib/kcp/test/e2e/kcp_test.go @@ -155,7 +155,7 @@ func testKcpIntegration(t *testing.T, name string, scope kubebindv1alpha2.Inform // Can assume that the last entry is now the cluster-id, grab it and // sanity check that it's not empty providerClusterID := providerClusterSplit[len(providerClusterSplit)-1] - require.NotEmpty(t, providerClusterID, "Retreived cluster id is empty, source URL: %s", providerCluster.Status.URL) + require.NotEmpty(t, providerClusterID, "Retrieved cluster id is empty, source URL: %s", providerCluster.Status.URL) // kube-bind process t.Log("Perform binding process with browser") diff --git a/docs/content/usage/.pages b/docs/content/usage/.pages index 69966eeef..0432813a3 100644 --- a/docs/content/usage/.pages +++ b/docs/content/usage/.pages @@ -1,2 +1,4 @@ nav: - - api-concepts.md \ No newline at end of file + - index.md + - api-concepts.md + - synchronization.md diff --git a/docs/content/usage/index.md b/docs/content/usage/index.md index 5267c604f..067456399 100644 --- a/docs/content/usage/index.md +++ b/docs/content/usage/index.md @@ -14,32 +14,39 @@ This section provides comprehensive documentation on how to use kube-bind's core kube-bind operates on three fundamental concepts: ### Service Provider + The cluster that **exports** APIs and resources, making them available for other clusters to consume. Service providers create templates and handle permission claims. -### Service Consumer +### Service Consumer + The cluster that **imports** and uses APIs from service providers. Consumers bind to templates and get access to resources through a secure, controlled process. ### Konnector Agent + The component that establishes and maintains the secure connection between provider and consumer clusters, synchronizing resources and handling permissions. ## Key API Types ### APIServiceExportTemplate + **Purpose**: Defines a reusable service template that groups related CRDs and permission claims. **Used by**: Service providers **Scope**: Template definition for multiple consumers ### APIServiceExport + **Purpose**: Represents an active export of a specific CRD to consumer clusters. **Used by**: Automatically created by konnector agents **Scope**: Per-CRD export instance ### APIServiceExportRequest + **Purpose**: Consumer's request to bind to a specific service template. **Used by**: Service consumers (via CLI/UI) **Scope**: Per-binding request ### APIServiceNamespace + **Purpose**: Manages namespace mapping and isolation between provider and consumer clusters. **Used by**: Automatically managed by konnector agents **Scope**: Per-namespace sync @@ -47,18 +54,22 @@ The component that establishes and maintains the secure connection between provi ## Documentation Structure ### [API Concepts](api-concepts.md) + Deep dive into the core API types, their relationships, and how they work together in the kube-bind ecosystem. ### [Template References](template-references.md) + Advanced guide for using dynamic resource selection through references in templates. ## Common Workflows ### For Service Providers + 1. **Create templates** defining what APIs and resources to export, including permission claims 2. **Implement service** to act on the synced/bound objects so it can be returned to the consumer/user. -### For Service Consumers +### For Service Consumers + 1. **Authenticate** to the kube-bind backend 1. **Discover available templates** through the web UI or CLI 2. **Request bindings** to specific templates @@ -66,8 +77,9 @@ Advanced guide for using dynamic resource selection through references in templa 4. **Use imported APIs** in their local cluster ### For Platform Operators + 1. **Deploy kube-bind infrastructure** on both provider and consumer sides (if using GitOps) -2. **Configure authentication** and security policies +2. **Configure authentication** and security policies 3. **Monitor connections** and resource synchronization ## Getting Started @@ -79,9 +91,9 @@ If you're new to kube-bind: 3. **Explore [Template References](template-references.md)** for advanced use cases 4. **Check the [Reference Documentation](../reference/)** for complete API specifications - The konnector agents establish a secure, authenticated connection that allows: + - **API schema synchronization** from provider to consumer - **Resource data flow** based on permission claims - **Namespace isolation** and mapping -- **Authentication and authorization** enforcement \ No newline at end of file +- **Authentication and authorization** enforcement diff --git a/docs/content/usage/synchronization.md b/docs/content/usage/synchronization.md new file mode 100644 index 000000000..ce2547d5c --- /dev/null +++ b/docs/content/usage/synchronization.md @@ -0,0 +1,173 @@ +--- +title: Resource Synchronization +description: | + Overview over the general resource synchronization logic and different isolation modes. +weight: 220 +--- + +# Resource Synchronization + +This document describes the way kube-bind synchronizes Kubernetes resources across clusters. + +## Overview + +kube-bind synchronizes objects between two kinds of clusters: + +* The **provider cluster** is run by a service provider and hosts a service, operator, or any other kind of Kubernetes API. This is also where there kube-bind **backend** is running in order to offer these APIs to consumers. +* The **consumer clusters** are where endusers consume services/APIs offered by providers. + +There exists a 1:n relationship, where one provider cluster can be connected to from many different consumer clusters. + +### Cluster Namespaces + +In kube-bind, each successful `bind` operation yields a new, so-called "cluster namespace" on the provider cluster. This namespace contains all kube-bind-related resources, like `APIServiceExports` or `BoundSchemas` and is communicated to the consumer by being included in the provided kubeconfig. + +By default the cluster namespaces are named `kube-bind-[random string]`, like `kube-bind-hd73s`. Their name also serves as a unique identifier for this "contract" between consumer and provider and is used in other places, for example as a prefix in the `prefixed` cluster isolation mode. + +### Sync Direction + +In the kube-bind architecture, the consumer clusters represent the source of truth (the actual desired state by the enduser) and the provider clusters merely contain copies of those objects. + +During the synchronization, + +* the **spec** (desired state) of an object is copied from the consumer cluster to the provider and +* the **status** (if any) is copied in the opposite direction, to the consumer. + +kube-bind will continuously watch the object and its copy on both clusters and update the other side as needed. + +### Connectivity + +The object synchronization logic lives in the kube-bind **konnector**, a Kubernetes agent that runs on each *consumer cluster*. It reads a local Secret that contains a kubeconfig pointing to **a specific namespace** on the *provider cluster*. In this namespace the konnector will find all resources describing the resources to sync (chiefly `APIServiceExports`, `APIServiceNamespaces` and `BoundSchemas`). + +!!! note + This kubeconfig is automatically generated as part of the `kubectl bind` handshake with a service provider. + +This design allows consumer clusters to be mostly firewalled off, but requires provider clusters to not only be reachable from all consumers, but also to ensure multiple consumers do not conflict with each other. This is achieved by a combination of RBAC and isolation modes, which are described further down in this document. + +### RBAC + +Great care must be taken to ensure multiple consumers do not collide with each other on a single provider cluster. To achieve this, kube-bind offers two different informer scoping options: + +* `namespaced` will make the konnector watch and inform on each relevant namespace on the provider cluster individually. +* `cluster` will instead make the konnector use a cluster-scoped (global) informer. This scoping method requires the konnector to have much wider permissions, but is more performant. + +Regardless of scoping mode, the konnector itself will take care to not overwrite/touch other consumers' objects. This however does not mean that an attacker with access to the konnector kubeconfig could not exploit an RBAC policy that is too wide. + +## Cluster Isolation + +This section outlines how kube-bind deals with differently scoped Kubernetes objects. + +In Kubernetes, an object can be either namespaced or cluster-scoped. This scope greatly affects how kube-bind processes the object. + +### Namespaced + +For namespaced objects (like a `Deployment`), kube-bind will map the consumer-side namespace to a unique, but random provider-side namespace: first the konnector will create an `APIServiceNamespace` object on the provider cluster (inside the cluster namespace), with the name of the consumer-side namespace (so a namespace `app1` will lead to an `APIServiceNamespace` `app1` in the cluster namespace). After this, the konnector waits until the backend has assigned this `APIServiceNamespace` a provider-side namespace to use. Once that namespace is present in the `APIServiceNamespace`'s status, the konnector will use it for all objects originating in the same consumer-side namespace. + +In YAML, this means + +```yaml +# consumer-side object + +apiVersion: provider.example.com/v1 +kind: MangoDB +metadata: + name: my-first-db + namespace: team1 +spec: + size: large +``` + +will lead to + +```yaml +# provider-side objects + +apiVersion: kube-bind.io/v1alpha2 +kind: APIServiceNamespace +metadata: + name: team1 # name of the namespace from the consumer + namespace: kube-bind-hd73d # cluster namespace +spec: {} +status: + # a common implementation in the backend is to construct the provider-side + # namespace by just concatenating the two values, like so: + namespace: kube-bind-hd73d-team1 + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: kube-bind-hd73d-team1 + +--- +apiVersion: provider.example.com/v1 +kind: MangoDB +metadata: + name: my-first-db + namespace: kube-bind-hd73d-team1 +spec: + size: large +``` + +!!! note + For natively namespaced resources (those that are namespaced in both the provider and consumer cluster), this is always the strategy being used by the konnector. The other isolation modes only influence how the konnector deals with cluster-scoped objects in the consumer cluster. + +### Cluster-Scoped + +Cluster-scoped objects require a different approach to isolation than namespaced objects. kube-bind offers three different so-called isolation strategies to deal with them. The strategy to use is configured globally via the `--cluster-scoped-isolation` CLI flag on the kube-bind backend and will from there affect all services offered by that backend. + +#### None Strategy + +The `none` strategy provides no consumer-separation at all. Any cluster-scoped object on the consumer side is copied 1:1 to the provider side. + +!!! warning + Due to the obvious downsides of this approach, `none` should be used only in special circumstances. + +#### Prefixed Strategy + +The `prefixed` strategy is kube-bind's default strategy and will use the name of the cluster namespace as a prefix for object names. + +In YAML, this means + +```yaml +# consumer-side +apiVersion: provider.example.com/v1 +kind: MangoDB +metadata: + name: my-first-db +spec: + size: large +``` + +will lead to + +```yaml +# provider-side +apiVersion: provider.example.com/v1 +kind: MangoDB +metadata: + name: kube-bind-hd73d-my-first-db +spec: + size: large +``` + +This strategy works well for separating objects, but + +* requires a broad RBAC policy, which will always grant too many permissions, and +* would technically break for Kubernetes objects with names close to the maximum allowed length (253 characters). + +#### Namespaced Strategy + +!!! note + Not to be confused with the strategy used for natively namespaced resources, described earlier. + +The `namespaced` strategy will convert cluster-scoped objects into namespaced ones. + +For this to work, the original CRD on the provider cluster has to be **namespaced**. The backend will turn it into a cluster-scoped CRD (stored in the `BoundSchema`), so on the consumer cluster, all objects are cluster-scoped. + +During synchronization, the konnector will then place each cluster-scoped object into the cluster namespace (`kube-bind-hd73d`) on the provider side. + +This strategy provides excellent isolation between consumers, but requires that the original CRD from the provider still makes sense to the consumer when it's suddenly cluster-scoped. For example, if references were to be used, especially to Secrets in the same namespace, this concept would get mangled to some degree during the synchronization. + +!!! warning + At the moment, when the backend is started with `--cluster-scoped-isolation=namespaced`, it will convert **all namespaced CRDs** in all ServiceExports to become cluster-scoped on the consumer side, even if you intended for a namespaced CRD to stay namespaced on both sides of the sync. diff --git a/docs/requirements.txt b/docs/requirements.txt index b613cf4d8..ae43afa1e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,3 +5,6 @@ mkdocs-macros-plugin==1.0.5 mkdocs-material==9.5.49 mkdocs-material-extensions==1.3.1 mkdocs-static-i18n==1.2.2 + +# https://github.com/mkdocs/mkdocs/issues/4032 +click<=8.2.1 diff --git a/pkg/konnector/controllers/cluster/serviceexport/cluster-scoped/utils.go b/pkg/konnector/controllers/cluster/serviceexport/cluster-scoped/utils.go deleted file mode 100644 index 6fe50b41a..000000000 --- a/pkg/konnector/controllers/cluster/serviceexport/cluster-scoped/utils.go +++ /dev/null @@ -1,165 +0,0 @@ -/* -Copyright 2023 The Kube Bind Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clusterscoped - -import ( - "errors" - "strings" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/types" -) - -// ClusterNsAnnotationKey is the annotation key to identify the cluster namespace that corresponds to -// provider side copy of a cluster-scoped object. Its value is the corresponding cluster namespace name. -const ClusterNsAnnotationKey = "kube-bind.io/cluster-namespace" - -// Prepend adds clusterNs and a dash before name. -func Prepend(name, clusterNs string) string { - return clusterNs + "-" + name -} - -// Behead tries to remove clusterNs and a dash that goes before name. -// If name doesn't start with clusterNs, name is returned unchanged. -func Behead(name, clusterNs string) string { - return strings.TrimPrefix(name, clusterNs+"-") -} - -// SwitchToUpstreamName switches the name of a downstream -// cluster-scoped object by prepending the cluster namespace name. -func SwitchToUpstreamName(obj *unstructured.Unstructured, clusterNs string) { - downstreamName := obj.GetName() - upstreamName := Prepend(downstreamName, clusterNs) - obj.SetName(upstreamName) -} - -// SwitchToDownstreamName switches the name of a upstream -// cluster-scoped object by removing the cluster namespace name as the prefix. -func SwitchToDownstreamName(obj *unstructured.Unstructured, clusterNs string) { - upstreamName := obj.GetName() - downstreamName := Behead(upstreamName, clusterNs) - obj.SetName(downstreamName) -} - -// InjectClusterNs injects the given cluster namespace (1) as an annotation, -// and (2) as a owner reference to the cluster namespace. -func InjectClusterNs(obj *unstructured.Unstructured, clusterNs, clusterNsUID string) error { - ans := obj.GetAnnotations() - existing, foundAn := ans[ClusterNsAnnotationKey] - if foundAn && existing != clusterNs { - return errors.New("mismatch between existing cluster namespace and given cluster namespace") - } - - ors := obj.GetOwnerReferences() - idx, foundOr := findOwnerReferenceToClusterNs(ors, clusterNs) - if foundOr && ors[idx].Name != clusterNs { - return errors.New("mismatch between existing cluster namespace and given cluster namespace") - } - - if !foundAn { - if ans == nil { - ans = map[string]string{} - } - ans[ClusterNsAnnotationKey] = clusterNs - obj.SetAnnotations(ans) - } - - if !foundOr { - ors = append(ors, metav1.OwnerReference{ - APIVersion: "v1", - Kind: "Namespace", - Name: clusterNs, - UID: types.UID(clusterNsUID), - }) - obj.SetOwnerReferences(ors) - } - - return nil -} - -// ExtractClusterNs extracts the corresponding cluster namespace name -// from a cluster-scoped object by reading the annotation. -func ExtractClusterNs(obj *unstructured.Unstructured) (string, error) { - ans := obj.GetAnnotations() - clusterNs, ok := ans[ClusterNsAnnotationKey] - if !ok { - return "", errors.New("cluster namespace annotation not found") - } - return clusterNs, nil -} - -// ClearClusterNs clears the given cluster namespace in a cluster-scoped object, -// including both the annotation and the owner reference. -func ClearClusterNs(obj *unstructured.Unstructured, clusterNs string) error { - ans := obj.GetAnnotations() - delete(ans, ClusterNsAnnotationKey) - obj.SetAnnotations(ans) - - ors := obj.GetOwnerReferences() - idx, foundOr := findOwnerReferenceToClusterNs(ors, clusterNs) - if foundOr { - ors[idx] = ors[len(ors)-1] - ors = ors[:len(ors)-1] - obj.SetOwnerReferences(ors) - } - - return nil -} - -// TranslateFromDownstream mutates a cluster-scoped object in place by injecting the cluster namespace -// and switching to its corresponding upstream name. -func TranslateFromDownstream(obj *unstructured.Unstructured, clusterNs, clusterNsUID string) error { - copy := obj.DeepCopy() - err := InjectClusterNs(copy, clusterNs, clusterNsUID) - if err != nil { - return err - } - SwitchToUpstreamName(copy, clusterNs) - *obj = *copy - return nil -} - -// TranslateFromUpstream mutates a cluster-scoped object in place by clearing the injected cluster namespace -// and switching to its corresponding downstream name. -func TranslateFromUpstream(obj *unstructured.Unstructured) error { - clusterNs, err := ExtractClusterNs(obj) - if err != nil { - return err - } - - copy := obj.DeepCopy() - err = ClearClusterNs(copy, clusterNs) - if err != nil { - return err - } - SwitchToDownstreamName(copy, clusterNs) - *obj = *copy - return nil -} - -func findOwnerReferenceToClusterNs(ors []metav1.OwnerReference, clusterNs string) (int, bool) { - if ors == nil { - return -1, false - } - for i, or := range ors { - if or.Kind == "Namespace" && or.Name == clusterNs { - return i, true - } - } - return -1, false -} diff --git a/pkg/konnector/controllers/cluster/serviceexport/cluster-scoped/utils_test.go b/pkg/konnector/controllers/cluster/serviceexport/cluster-scoped/utils_test.go deleted file mode 100644 index 5f4e2a7ab..000000000 --- a/pkg/konnector/controllers/cluster/serviceexport/cluster-scoped/utils_test.go +++ /dev/null @@ -1,204 +0,0 @@ -/* -Copyright 2023 The Kube Bind Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clusterscoped - -import ( - "testing" - - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" -) - -func TestSwitchToUpstreamName(t *testing.T) { - tests := []struct { - name string - clusterNs string - down string - expected string - }{ - { - name: "2up", - clusterNs: "kube-bind-zlp9m", - down: "example-foo", - expected: "kube-bind-zlp9m-example-foo", - }, - } - for _, tt := range tests { - obj := unstructured.Unstructured{} - obj.SetName(tt.down) - SwitchToUpstreamName(&obj, tt.clusterNs) - t.Run(tt.name, func(t *testing.T) { - if actual := obj.GetName(); actual != tt.expected { - t.Error("SwitchToUpstreamName() error", "expected", tt.expected, "actual", actual) - } - }) - } -} - -func TestSwitchToDownstreamName(t *testing.T) { - tests := []struct { - name string - clusterNs string - up string - expected string - }{ - { - name: "2down", - clusterNs: "kube-bind-zlp9m", - up: "kube-bind-zlp9m-example-foo", - expected: "example-foo", - }, - } - for _, tt := range tests { - obj := unstructured.Unstructured{} - obj.SetName(tt.up) - SwitchToDownstreamName(&obj, tt.clusterNs) - t.Run(tt.name, func(t *testing.T) { - if actual := obj.GetName(); actual != tt.expected { - t.Error("SwitchToUpstreamName() error", "expected", tt.expected, "actual", actual) - } - }) - } -} - -func TestInjectClusterNs(t *testing.T) { - tests := []struct { - name string - obj unstructured.Unstructured - clusterNs string - clusterNsUID string - expected string - wantErr bool - }{ - { - name: "noExistingClusterNs", - obj: unstructured.Unstructured{}, - clusterNs: "kube-bind-zlp9m", - clusterNsUID: "real-identity", - expected: "kube-bind-zlp9m", - wantErr: false, - }, - { - name: "oneExistingClusterNs", - obj: newObjectWithClusterNs("kube-bind-zlp9m"), - clusterNs: "kube-bind-s85lc", - clusterNsUID: "real-indentity", - expected: "kube-bind-zlp9m", - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := InjectClusterNs(&tt.obj, tt.clusterNs, tt.clusterNsUID); (err != nil) != tt.wantErr { - actual := tt.obj.GetAnnotations()[ClusterNsAnnotationKey] - t.Error("InjectClusterNs() error", "error", err, "expected", tt.expected, "actual", actual) - } else if err == nil { - actual := tt.obj.GetAnnotations()[ClusterNsAnnotationKey] - require.Equal(t, tt.expected, actual) - } else { - t.Log("expected error", err) - } - }) - } -} - -func TestExtractClusterNs(t *testing.T) { - tests := []struct { - name string - obj unstructured.Unstructured - expected string - wantErr bool - }{ - { - name: "oneExistingClusterNs", - obj: newObjectWithClusterNs("kube-bind-zlp9m"), - expected: "kube-bind-zlp9m", - wantErr: false, - }, - { - name: "noExistingClusterNs", - obj: unstructured.Unstructured{}, - expected: "", - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if actual, err := ExtractClusterNs(&tt.obj); (err != nil) != tt.wantErr { - t.Error("ExtractClusterNs() error", "error", err, "expected", tt.expected, "actual", actual) - } else if err == nil { - require.Equal(t, tt.expected, actual) - } else { - t.Log("expected error", err) - } - }) - } -} - -func TestClearClusterNs(t *testing.T) { - tests := []struct { - name string - obj unstructured.Unstructured - clusterNs string - expected int - wantErr bool - }{ - { - name: "oneExistingClusterNs", - obj: newObjectWithClusterNs("kube-bind-zlp9m"), - clusterNs: "kube-bind-zlp9m", - expected: 0, - wantErr: false, - }, - { - name: "noExistingClusterNs", - obj: unstructured.Unstructured{}, - clusterNs: "not-applicable", - expected: 0, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := ClearClusterNs(&tt.obj, tt.clusterNs); (err != nil) != tt.wantErr { - actual := len(tt.obj.GetOwnerReferences()) - t.Error("ClearClusterNs() error", "error", err, "expected", tt.expected, "actual", actual) - } else if err == nil { - actual := len(tt.obj.GetOwnerReferences()) - require.Equal(t, tt.expected, actual) - } else { - t.Log("expected error", err) - } - }) - } -} - -func newObjectWithClusterNs(name string) unstructured.Unstructured { - obj := unstructured.Unstructured{} - ans := map[string]string{ - ClusterNsAnnotationKey: name, - } - obj.SetAnnotations(ans) - ors := []metav1.OwnerReference{{ - APIVersion: "v1", - Kind: "Namespace", - Name: name, - }} - obj.SetOwnerReferences(ors) - return obj -} diff --git a/pkg/konnector/controllers/cluster/serviceexport/isolation/namespaced.go b/pkg/konnector/controllers/cluster/serviceexport/isolation/namespaced.go new file mode 100644 index 000000000..11836a866 --- /dev/null +++ b/pkg/konnector/controllers/cluster/serviceexport/isolation/namespaced.go @@ -0,0 +1,70 @@ +/* +Copyright 2025 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package isolation + +import ( + "context" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +type namespacedStrategy struct { + clusterNamespace string +} + +// NewNamespaced returns an isolation strategy that transforms cluster-scoped +// objects from the consumer cluster into namespaced objects on the provider +// cluster. All objects will be placed in the cluster namespace. +func NewNamespaced(clusterNamespace string) Strategy { + return &namespacedStrategy{ + clusterNamespace: clusterNamespace, + } +} + +func (s *namespacedStrategy) ToProviderKey(consumerKey types.NamespacedName) (*types.NamespacedName, error) { + return &types.NamespacedName{ + Namespace: s.clusterNamespace, + Name: consumerKey.Name, + }, nil +} + +func (s *namespacedStrategy) EnsureProviderKey(_ context.Context, consumerKey types.NamespacedName) (*types.NamespacedName, error) { + return s.ToProviderKey(consumerKey) +} + +func (s *namespacedStrategy) ToConsumerKey(providerKey types.NamespacedName) (*types.NamespacedName, error) { + if providerKey.Namespace != s.clusterNamespace { + return nil, nil + } + + return &types.NamespacedName{Name: providerKey.Name}, nil +} + +func (s *namespacedStrategy) MutateMetadataAndSpec(consumerObj *unstructured.Unstructured, providerKey types.NamespacedName) error { + consumerObj.SetName(providerKey.Name) + consumerObj.SetNamespace(providerKey.Namespace) + + return nil +} + +func (s *namespacedStrategy) MutateStatus(providerObj *unstructured.Unstructured, consumerKey types.NamespacedName) error { + providerObj.SetName(consumerKey.Name) + providerObj.SetNamespace(consumerKey.Namespace) + + return nil +} diff --git a/pkg/konnector/controllers/cluster/serviceexport/isolation/none.go b/pkg/konnector/controllers/cluster/serviceexport/isolation/none.go new file mode 100644 index 000000000..4bd046b9a --- /dev/null +++ b/pkg/konnector/controllers/cluster/serviceexport/isolation/none.go @@ -0,0 +1,93 @@ +/* +Copyright 2025 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package isolation + +import ( + "context" + "errors" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +type noneStrategy struct { + clusterNamespace string + clusterNamespaceUID string +} + +// NewNone returns an isolation strategy for cluster-scoped objects that will not +// change anything about their name. Using this strategy will most likely lead to +// naming conflicts between consumers and should only be used with great care. +// Since this strategy will lead to cluster-scoped objects on the provide side, +// it will add an owner reference to each, pointing to the cluster namespace. +func NewNone(clusterNamespace string, clusterNamespaceUID string) Strategy { + return &noneStrategy{ + clusterNamespace: clusterNamespace, + clusterNamespaceUID: clusterNamespaceUID, + } +} + +func (*noneStrategy) ToProviderKey(consumerKey types.NamespacedName) (*types.NamespacedName, error) { + return &consumerKey, nil +} + +func (s *noneStrategy) EnsureProviderKey(_ context.Context, consumerKey types.NamespacedName) (*types.NamespacedName, error) { + return s.ToProviderKey(consumerKey) +} + +func (*noneStrategy) ToConsumerKey(providerKey types.NamespacedName) (*types.NamespacedName, error) { + return &providerKey, nil +} + +func (s *noneStrategy) MutateMetadataAndSpec(consumerObj *unstructured.Unstructured, providerKey types.NamespacedName) error { + return setOwnerReference(consumerObj, s.clusterNamespace, s.clusterNamespaceUID) +} + +func (*noneStrategy) MutateStatus(providerObj *unstructured.Unstructured, consumerKey types.NamespacedName) error { + return nil +} + +func setOwnerReference(obj *unstructured.Unstructured, clusterNamespace string, clusterNamespaceUID string) error { + ownerRefs := obj.GetOwnerReferences() + ownerRef := findOwnerReference(ownerRefs, clusterNamespace) + if ownerRef != nil && ownerRef.Name != clusterNamespace { + return errors.New("mismatch between existing cluster namespace and given cluster namespace") + } + + if ownerRef == nil { + ownerRefs = append(ownerRefs, metav1.OwnerReference{ + APIVersion: "v1", + Kind: "Namespace", + Name: clusterNamespace, + UID: types.UID(clusterNamespaceUID), + }) + obj.SetOwnerReferences(ownerRefs) + } + + return nil +} + +func findOwnerReference(refs []metav1.OwnerReference, clusterNamespace string) *metav1.OwnerReference { + for _, ref := range refs { + if ref.Kind == "Namespace" && ref.Name == clusterNamespace { + return &ref + } + } + + return nil +} diff --git a/pkg/konnector/controllers/cluster/serviceexport/isolation/prefixed.go b/pkg/konnector/controllers/cluster/serviceexport/isolation/prefixed.go new file mode 100644 index 000000000..711321547 --- /dev/null +++ b/pkg/konnector/controllers/cluster/serviceexport/isolation/prefixed.go @@ -0,0 +1,85 @@ +/* +Copyright 2025 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package isolation + +import ( + "context" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +type prefixedStrategy struct { + // On a technical level, this is the same as the basic "none" behaviour, just + // with additional name mutation; since we want to re-use the other code for + // cluster-scoped objects *on the provider side*, we extend the none strategy. + noneStrategy + + clusterNamespace string +} + +// NewPrefixed returns an isolation strategy for cluster-scoped objects where each +// object on the provider cluster gets the name of the cluster namespace prepended +// to their name (i.e. turning "my-obj" into "kube-bind-abc123-my-obj"). This is +// effective and easy since no scoping changes need to be accounted for (i.e. the +// BoundSchema does not need to be adjusted), but could theoretically cause problems +// for objects with very long names that do not have enough room for such a prefix. +func NewPrefixed(clusterNamespace string, clusterNamespaceUID string) Strategy { + return &prefixedStrategy{ + noneStrategy: noneStrategy{ + clusterNamespace: clusterNamespace, + clusterNamespaceUID: clusterNamespaceUID, + }, + clusterNamespace: clusterNamespace, + } +} + +func (s *prefixedStrategy) ToProviderKey(consumerKey types.NamespacedName) (*types.NamespacedName, error) { + return &types.NamespacedName{ + Name: s.clusterNamespace + "-" + consumerKey.Name, + }, nil +} + +func (s *prefixedStrategy) EnsureProviderKey(_ context.Context, consumerKey types.NamespacedName) (*types.NamespacedName, error) { + return s.ToProviderKey(consumerKey) +} + +func (s *prefixedStrategy) ToConsumerKey(providerKey types.NamespacedName) (*types.NamespacedName, error) { + prefix := s.clusterNamespace + "-" + if !strings.HasPrefix(providerKey.Name, prefix) { + return nil, nil + } + + return &types.NamespacedName{ + Name: strings.TrimPrefix(providerKey.Name, prefix), + }, nil +} + +func (s *prefixedStrategy) MutateMetadataAndSpec(consumerObj *unstructured.Unstructured, providerKey types.NamespacedName) error { + consumerObj.SetName(providerKey.Name) + consumerObj.SetNamespace(providerKey.Namespace) + + return s.noneStrategy.MutateMetadataAndSpec(consumerObj, providerKey) +} + +func (s *prefixedStrategy) MutateStatus(providerObj *unstructured.Unstructured, consumerKey types.NamespacedName) error { + providerObj.SetName(consumerKey.Name) + providerObj.SetNamespace(consumerKey.Namespace) + + return s.noneStrategy.MutateStatus(providerObj, consumerKey) +} diff --git a/pkg/konnector/controllers/cluster/serviceexport/isolation/prefixed_test.go b/pkg/konnector/controllers/cluster/serviceexport/isolation/prefixed_test.go new file mode 100644 index 000000000..654e33d7f --- /dev/null +++ b/pkg/konnector/controllers/cluster/serviceexport/isolation/prefixed_test.go @@ -0,0 +1,84 @@ +/* +Copyright 2023 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package isolation + +import ( + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/types" +) + +func TestPrefixedToProviderKey(t *testing.T) { + tests := []struct { + name string + providerNs string + objectName string + expected string + }{ + { + name: "basic testcase", + providerNs: "kube-bind-zlp9m", + objectName: "example-foo", + expected: "kube-bind-zlp9m-example-foo", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + strategy := NewPrefixed(tt.providerNs, "irrelevant-uid") + + result, err := strategy.ToProviderKey(types.NamespacedName{Name: tt.objectName}) + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "", result.Namespace) + require.Equal(t, tt.expected, result.Name) + }) + } +} + +func TestPrefixedToConsumerKey(t *testing.T) { + tests := []struct { + name string + providerNs string + objectName string + expected string + }{ + { + name: "basic testcase", + providerNs: "kube-bind-zlp9m", + objectName: "kube-bind-zlp9m-example-foo", + expected: "example-foo", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + strategy := NewPrefixed(tt.providerNs, "irrelevant-uid") + + result, err := strategy.ToConsumerKey(types.NamespacedName{Name: tt.objectName}) + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "", result.Namespace) + require.Equal(t, tt.expected, result.Name) + }) + } +} diff --git a/pkg/konnector/controllers/cluster/serviceexport/isolation/servicenamespaced.go b/pkg/konnector/controllers/cluster/serviceexport/isolation/servicenamespaced.go new file mode 100644 index 000000000..7099b18ae --- /dev/null +++ b/pkg/konnector/controllers/cluster/serviceexport/isolation/servicenamespaced.go @@ -0,0 +1,163 @@ +/* +Copyright 2025 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package isolation + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + + "github.com/kube-bind/kube-bind/pkg/indexers" + "github.com/kube-bind/kube-bind/pkg/konnector/controllers/dynamic" + kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" + bindlisters "github.com/kube-bind/kube-bind/sdk/client/listers/kubebind/v1alpha2" +) + +type ServiceNamespacedStrategy struct { + clusterNamespace string + serviceNamespaceInformer dynamic.Informer[bindlisters.APIServiceNamespaceLister] + serviceNamespaceCreator func(ctx context.Context, name string) (*kubebindv1alpha2.APIServiceNamespace, error) +} + +// NewServiceNamespaced returns the one and only valid isolation strategy for +// namespaced objects. It is special in a sense that it does not map consumer +// namespaces 1:1 to provider namespaces, but uses APIServiceNamespace objects +// to request the backend to assign a namespace on the provider cluster. This +// strategy must not be used for cluster-scoped resources. +func NewServiceNamespaced( + clusterNamespace string, + serviceNamespaceInformer dynamic.Informer[bindlisters.APIServiceNamespaceLister], + serviceNamespaceCreator func(ctx context.Context, name string) (*kubebindv1alpha2.APIServiceNamespace, error), +) Strategy { + return &ServiceNamespacedStrategy{ + clusterNamespace: clusterNamespace, + serviceNamespaceInformer: serviceNamespaceInformer, + serviceNamespaceCreator: serviceNamespaceCreator, + } +} + +func (s *ServiceNamespacedStrategy) ProviderNamespace(consumerNamespace string) (string, error) { + sn, err := s.serviceNamespaceInformer.Lister().APIServiceNamespaces(s.clusterNamespace).Get(consumerNamespace) + if err != nil { + if !errors.IsNotFound(err) { + return "", err + } + + return "", nil + } + + return sn.Status.Namespace, nil +} + +// ConsumerNamespace returns the namespace on the consumer cluster that owns the +// API namespace on the provider cluster. This function effectively performs a +// reverse lookup using the APIServiceNamespace's status. +// This function can return an empty name if the APIServiceNamespace is not ready yet. +func (s *ServiceNamespacedStrategy) ConsumerNamespace(namespace string) (string, error) { + sns, err := s.serviceNamespaceInformer.Informer().GetIndexer().ByIndex(indexers.ServiceNamespaceByNamespace, namespace) + if err != nil { + return "", err + } + + for _, obj := range sns { + sns := obj.(*kubebindv1alpha2.APIServiceNamespace) + if sns.Namespace == s.clusterNamespace { + return sns.Name, nil + } + } + + return "", nil +} + +func (s *ServiceNamespacedStrategy) ToProviderKey(consumerKey types.NamespacedName) (*types.NamespacedName, error) { + providerNs, err := s.ProviderNamespace(consumerKey.Namespace) + if err != nil { + return nil, err + } + + // not ready yet + if providerNs == "" { + return nil, nil + } + + return &types.NamespacedName{ + Namespace: providerNs, + Name: consumerKey.Name, + }, nil +} + +func (s *ServiceNamespacedStrategy) EnsureProviderKey(ctx context.Context, consumerKey types.NamespacedName) (*types.NamespacedName, error) { + sn, err := s.serviceNamespaceInformer.Lister().APIServiceNamespaces(s.clusterNamespace).Get(consumerKey.Namespace) + if err != nil && !errors.IsNotFound(err) { + return nil, err + } + + logger := klog.FromContext(ctx).WithValues("namespace", consumerKey.Namespace) + + if errors.IsNotFound(err) { + logger.V(1).Info("creating APIServiceNamespace") + sn, err = s.serviceNamespaceCreator(ctx, consumerKey.Namespace) + if err != nil { + return nil, err + } + } + + if sn.Status.Namespace == "" { + // note: the service provider might implement this synchronously in admission. if so, we can skip the requeue. + logger.V(1).Info("waiting for APIServiceNamespace to be ready") + return nil, nil + } + + return &types.NamespacedName{ + Namespace: sn.Status.Namespace, + Name: consumerKey.Name, + }, nil +} + +func (s *ServiceNamespacedStrategy) ToConsumerKey(providerKey types.NamespacedName) (*types.NamespacedName, error) { + consumerNs, err := s.ConsumerNamespace(providerKey.Namespace) + if err != nil { + return nil, err + } + + // not ready yet + if consumerNs == "" { + return nil, nil + } + + return &types.NamespacedName{ + Namespace: consumerNs, + Name: providerKey.Name, + }, nil +} + +func (s *ServiceNamespacedStrategy) MutateMetadataAndSpec(consumerObj *unstructured.Unstructured, providerKey types.NamespacedName) error { + consumerObj.SetName(providerKey.Name) + consumerObj.SetNamespace(providerKey.Namespace) + + return nil +} + +func (s *ServiceNamespacedStrategy) MutateStatus(providerObj *unstructured.Unstructured, consumerKey types.NamespacedName) error { + providerObj.SetName(consumerKey.Name) + providerObj.SetNamespace(consumerKey.Namespace) + + return nil +} diff --git a/pkg/konnector/controllers/cluster/serviceexport/isolation/strategy.go b/pkg/konnector/controllers/cluster/serviceexport/isolation/strategy.go new file mode 100644 index 000000000..73e65f2b5 --- /dev/null +++ b/pkg/konnector/controllers/cluster/serviceexport/isolation/strategy.go @@ -0,0 +1,58 @@ +/* +Copyright 2025 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package isolation + +import ( + "context" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// Strategy implements the namespace/name translation for Kubernetes objects when +// they are copied from one cluster to another. Not every strategy is necessarily +// applicable to all circumstances (strategies can be for cluster-scoped or +// namespaced objects only, check their documentation). +type Strategy interface { + // ToProviderKey translates the namespace/name (collectively called "key") from + // the consumer side to the provider side. This function can return a nil key + // for invalid/foreign objects, so callers need to be aware. + ToProviderKey(consumerKey types.NamespacedName) (*types.NamespacedName, error) + + // ToConsumerKey translates the namespace/name (collectively called "key") from + // the provider side to the consumer side. This function can return a nil key + // for invalid/foreign objects, so callers need to be aware. + ToConsumerKey(providerKey types.NamespacedName) (*types.NamespacedName, error) + + // EnsureProviderKey is very similar to ToProviderKey, but may make changes + // on the provider side and return a nil key in case the target key is simply + // not ready yet. This is most often the case when waiting for the backend + // to assign a namespace for an APIServiceNamespace. + EnsureProviderKey(ctx context.Context, consumerKey types.NamespacedName) (*types.NamespacedName, error) + + // MutateMetadataAndSpec mutates the object's content (its main resource) when + // syncing from the consumer side to the provide side. This function is also + // responsible for applying the translated keys (from ToProviderKey). + MutateMetadataAndSpec(consumerObj *unstructured.Unstructured, providerKey types.NamespacedName) error + + // MutateStatus is the opposite of MutateMetadataAndSpec and might mutate the + // object status (status subresource) when syncing it back from the provider + // cluster to the consumer cluster. This function is also responsible for + // applying the consumer key to "rename" the object back to its original name + // on the consumer cluster. + MutateStatus(providerObj *unstructured.Unstructured, consumerKey types.NamespacedName) error +} diff --git a/pkg/konnector/controllers/cluster/serviceexport/serviceexport_reconcile.go b/pkg/konnector/controllers/cluster/serviceexport/serviceexport_reconcile.go index 768bf4c80..c1e6856f6 100644 --- a/pkg/konnector/controllers/cluster/serviceexport/serviceexport_reconcile.go +++ b/pkg/konnector/controllers/cluster/serviceexport/serviceexport_reconcile.go @@ -38,6 +38,7 @@ import ( "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/claimedresources" "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/claimedresourcesnamespaces" + "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/isolation" "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/multinsinformer" "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/spec" "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/status" @@ -46,6 +47,7 @@ import ( kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" conditionsapi "github.com/kube-bind/kube-bind/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" "github.com/kube-bind/kube-bind/sdk/apis/third_party/conditions/util/conditions" + bindclient "github.com/kube-bind/kube-bind/sdk/client/clientset/versioned" bindlisters "github.com/kube-bind/kube-bind/sdk/client/listers/kubebind/v1alpha2" ) @@ -219,6 +221,7 @@ func (r *reconciler) ensureControllerForSchema(ctx context.Context, export *kube consumerInf := dynamicinformer.NewDynamicSharedInformerFactory(dynamicConsumerClient, time.Minute*30) var providerInf multinsinformer.GetterInformer + if schema.Spec.Scope == apiextensionsv1.ClusterScoped || schema.Spec.InformerScope == kubebindv1alpha2.ClusterScope { factory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicProviderClient, time.Minute*30) factory.ForResource(gvr).Lister() // wire the GVR up in the informer factory @@ -239,7 +242,43 @@ func (r *reconciler) ensureControllerForSchema(ctx context.Context, export *kube } } + var isolationStrategy isolation.Strategy + switch { + case schema.Spec.Scope == apiextensionsv1.NamespaceScoped: + providerBindClient, err := bindclient.NewForConfig(r.providerConfig) + if err != nil { + return err + } + + isolationStrategy = isolation.NewServiceNamespaced( + r.providerNamespace, + r.serviceNamespaceInformer, + func(ctx context.Context, name string) (*kubebindv1alpha2.APIServiceNamespace, error) { + sn := &kubebindv1alpha2.APIServiceNamespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: r.providerNamespace, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(export, kubebindv1alpha2.SchemeGroupVersion.WithKind("APIServiceExport")), + }, + }, + } + + return providerBindClient.KubeBindV1alpha2().APIServiceNamespaces(sn.Namespace).Create(ctx, sn, metav1.CreateOptions{}) + }) + + case export.Spec.ClusterScopedIsolation == kubebindv1alpha2.IsolationNone: + isolationStrategy = isolation.NewNone(r.providerNamespace, providerNamespaceUID) + + case export.Spec.ClusterScopedIsolation == kubebindv1alpha2.IsolationPrefixed: + isolationStrategy = isolation.NewPrefixed(r.providerNamespace, providerNamespaceUID) + + case export.Spec.ClusterScopedIsolation == kubebindv1alpha2.IsolationNamespaced: + isolationStrategy = isolation.NewNamespaced(r.providerNamespace) + } + specCtrl, err := spec.NewController( + isolationStrategy, export, // pass the export to establish owner references on ServiceNamespace creation gvr, r.providerNamespace, @@ -256,6 +295,7 @@ func (r *reconciler) ensureControllerForSchema(ctx context.Context, export *kube } statusCtrl, err := status.NewController( + isolationStrategy, gvr, r.providerNamespace, providerNamespaceUID, @@ -364,7 +404,7 @@ func (r *reconciler) ensureControllerForPermissionClaim( dynamicProviderClient := dynamicclient.NewForConfigOrDie(r.providerConfig) dynamicConsumerClient := dynamicclient.NewForConfigOrDie(r.consumerConfig) - // Create consumer informer factory. This is always unfiltered, as we might be geeting obejcts from referece, + // Create consumer informer factory. This is always unfiltered, as we might be getting objcts from reference, // label or named. We need to see all objects to determine if they are claimed. defaultConsumerInf := dynamicinformer.NewDynamicSharedInformerFactory(dynamicConsumerClient, time.Minute*30) diff --git a/pkg/konnector/controllers/cluster/serviceexport/spec/spec_controller.go b/pkg/konnector/controllers/cluster/serviceexport/spec/spec_controller.go index a6a54b129..f3c5ff26b 100644 --- a/pkg/konnector/controllers/cluster/serviceexport/spec/spec_controller.go +++ b/pkg/konnector/controllers/cluster/serviceexport/spec/spec_controller.go @@ -38,8 +38,7 @@ import ( "k8s.io/klog/v2" "k8s.io/utils/ptr" - "github.com/kube-bind/kube-bind/pkg/indexers" - clusterscoped "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/cluster-scoped" + "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/isolation" "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/multinsinformer" "github.com/kube-bind/kube-bind/pkg/konnector/controllers/dynamic" kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" @@ -55,6 +54,7 @@ const ( // NewController returns a new controller reconciling downstream objects to upstream. func NewController( + isolationStrategy isolation.Strategy, apiServiceExport *kubebindv1alpha2.APIServiceExport, // used to establish owner references when create happens from the consumer side. gvr schema.GroupVersionResource, providerNamespace string, @@ -88,8 +88,9 @@ func NewController( c := &controller{ queue: queue, - consumerClient: consumerClient, - providerClient: providerClient, + consumerClient: consumerClient, + providerClient: providerClient, + providerNamespace: providerNamespace, consumerDynamicLister: dynamicConsumerLister, consumerDynamicIndexer: consumerDynamicInformer.Informer().GetIndexer(), @@ -99,8 +100,8 @@ func NewController( serviceNamespaceInformer: serviceNamespaceInformer, reconciler: reconciler{ - providerNamespace: providerNamespace, - apiServiceExport: apiServiceExport, + isolationStrategy: isolationStrategy, + clusterNamespace: providerNamespace, getServiceNamespace: func(name string) (*kubebindv1alpha2.APIServiceNamespace, error) { return serviceNamespaceInformer.Lister().APIServiceNamespaces(providerNamespace).Get(name) @@ -109,77 +110,28 @@ func NewController( return providerBindClient.KubeBindV1alpha2().APIServiceNamespaces(providerNamespace).Create(ctx, sn, metav1.CreateOptions{}) }, getProviderObject: func(ns, name string) (*unstructured.Unstructured, error) { - if ns != "" { - obj, err := providerDynamicInformer.Get(ns, name) - if err != nil { - return nil, err - } - return obj.(*unstructured.Unstructured), nil - } - got, err := providerDynamicInformer.Get(ns, clusterscoped.Prepend(name, providerNamespace)) - if err != nil { - return nil, err - } - obj := got.(*unstructured.Unstructured).DeepCopy() - err = clusterscoped.TranslateFromUpstream(obj) + got, err := providerDynamicInformer.Get(ns, name) if err != nil { return nil, err } - return obj, nil + return got.(*unstructured.Unstructured).DeepCopy(), nil }, createProviderObject: func(ctx context.Context, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { - if ns := obj.GetNamespace(); ns != "" { - copy := obj.DeepCopy() - err := clusterscoped.InjectClusterNs(copy, providerNamespace, providerNamespaceUID) - if err != nil { - return nil, err - } - return providerClient.Resource(gvr).Namespace(copy.GetNamespace()).Create(ctx, copy, metav1.CreateOptions{}) - } - err := clusterscoped.TranslateFromDownstream(obj, providerNamespace, providerNamespaceUID) - if err != nil { - return nil, err - } - created, err := providerClient.Resource(gvr).Namespace(obj.GetNamespace()).Create(ctx, obj, metav1.CreateOptions{}) - if err != nil { - return nil, err - } - err = clusterscoped.TranslateFromUpstream(created) - if err != nil { - return nil, err - } - return created, nil + return providerClient.Resource(gvr).Namespace(obj.GetNamespace()).Create(ctx, obj, metav1.CreateOptions{}) }, - updateProviderObject: func(ctx context.Context, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { - ns := obj.GetNamespace() - if ns == "" { - if err := clusterscoped.TranslateFromDownstream(obj, providerNamespace, providerNamespaceUID); err != nil { - return nil, err - } - } + updateProviderObject: func(ctx context.Context, obj *unstructured.Unstructured) error { + obj.SetManagedFields(nil) // server side apply does not want this + data, err := json.Marshal(obj.Object) if err != nil { - return nil, err + return err } - patched, err := providerClient.Resource(gvr).Namespace(obj.GetNamespace()).Patch(ctx, + _, err = providerClient.Resource(gvr).Namespace(obj.GetNamespace()).Patch(ctx, obj.GetName(), types.ApplyPatchType, data, metav1.PatchOptions{FieldManager: applyManager, Force: ptr.To(true)}, ) - if err != nil { - return nil, err - } - if ns == "" { - err = clusterscoped.TranslateFromUpstream(patched) - if err != nil { - return nil, err - } - return patched, nil - } - return patched, nil + return err }, deleteProviderObject: func(ctx context.Context, ns, name string) error { - if ns == "" { - name = clusterscoped.Prepend(name, providerNamespace) - } return providerClient.Resource(gvr).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{}) }, updateConsumerObject: func(ctx context.Context, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { @@ -231,6 +183,8 @@ func NewController( type controller struct { queue workqueue.TypedRateLimitingInterface[string] + providerNamespace string + consumerClient dynamicclient.Interface providerClient dynamicclient.Interface @@ -251,49 +205,39 @@ func (c *controller) enqueueConsumer(logger klog.Logger, obj any) { return } - logger.V(2).Info("queueing Unstructured", "key", key) + logger.V(2).Info("queueing Unstructured", "queued", key, "reason", "consumerObject") c.queue.Add(key) } func (c *controller) enqueueProvider(logger klog.Logger, obj any) { - upstreamKey, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) + providerKey, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) if err != nil { runtime.HandleError(err) return } - ns, name, err := cache.SplitMetaNamespaceKey(upstreamKey) + ns, name, err := cache.SplitMetaNamespaceKey(providerKey) if err != nil { runtime.HandleError(err) return } - if ns != "" { - sns, err := c.serviceNamespaceInformer.Informer().GetIndexer().ByIndex(indexers.ServiceNamespaceByNamespace, ns) - if err != nil { - if !errors.IsNotFound(err) { - runtime.HandleError(err) - } - return - } - for _, obj := range sns { - sn := obj.(*kubebindv1alpha2.APIServiceNamespace) - if sn.Namespace == c.providerNamespace { - key := fmt.Sprintf("%s/%s", sn.Name, name) - logger.V(2).Info("queueing Unstructured", "key", key) - c.queue.Add(key) - return - } + consumerKey, err := c.isolationStrategy.ToConsumerKey(types.NamespacedName{ + Namespace: ns, + Name: name, + }) + if err != nil { + if !errors.IsNotFound(err) { + runtime.HandleError(err) } return } - if clusterscoped.Behead(upstreamKey, c.providerNamespace) == upstreamKey { - logger.V(3).Info("skipping because consumer mismatch", "upstreamKey", upstreamKey) + if consumerKey == nil { return } - downstreamKey := clusterscoped.Behead(upstreamKey, c.providerNamespace) - logger.V(2).Info("queueing Unstructured", "key", downstreamKey) - c.queue.Add(downstreamKey) + + logger.V(2).Info("queueing Unstructured", "queued", consumerKey, "reason", "providerObject") + c.queue.Add(consumerKey.String()) } func (c *controller) enqueueServiceNamespace(logger klog.Logger, obj any) { @@ -322,7 +266,7 @@ func (c *controller) enqueueServiceNamespace(logger klog.Logger, obj any) { runtime.HandleError(err) continue } - logger.V(2).Info("queueing Unstructured", "key", key, "reason", "APIServiceNamespace", "ServiceNamespaceKey", snKey) + logger.V(2).Info("queueing Unstructured", "queued", key, "reason", "APIServiceNamespace", "ServiceNamespaceKey", snKey) c.queue.Add(key) } } diff --git a/pkg/konnector/controllers/cluster/serviceexport/spec/spec_reconcile.go b/pkg/konnector/controllers/cluster/serviceexport/spec/spec_reconcile.go index 7acc317c9..4ee3a250c 100644 --- a/pkg/konnector/controllers/cluster/serviceexport/spec/spec_reconcile.go +++ b/pkg/konnector/controllers/cluster/serviceexport/spec/spec_reconcile.go @@ -19,28 +19,32 @@ package spec import ( "context" "encoding/json" + "errors" "fmt" "reflect" + "slices" "time" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" + "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/isolation" + konnectortypes "github.com/kube-bind/kube-bind/pkg/konnector/types" kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" ) type reconciler struct { - providerNamespace string - apiServiceExport *kubebindv1alpha2.APIServiceExport // used to establish owner references when create happens from the consumer side. + isolationStrategy isolation.Strategy + clusterNamespace string getServiceNamespace func(name string) (*kubebindv1alpha2.APIServiceNamespace, error) createServiceNamespace func(ctx context.Context, sn *kubebindv1alpha2.APIServiceNamespace) (*kubebindv1alpha2.APIServiceNamespace, error) getProviderObject func(ns, name string) (*unstructured.Unstructured, error) createProviderObject func(ctx context.Context, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) - updateProviderObject func(ctx context.Context, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) + updateProviderObject func(ctx context.Context, obj *unstructured.Unstructured) error deleteProviderObject func(ctx context.Context, ns, name string) error updateConsumerObject func(ctx context.Context, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) @@ -51,48 +55,27 @@ type reconciler struct { // reconcile syncs downstream objects (metadata and spec) with upstream objects. func (r *reconciler) reconcile(ctx context.Context, obj *unstructured.Unstructured) error { logger := klog.FromContext(ctx) - if r.apiServiceExport == nil { // Should never happen, but we check to make sure we dont regress in the future. - return fmt.Errorf("internal error: apiServiceExport is nil") - } - - ns := obj.GetNamespace() - if ns != "" { - sn, err := r.getServiceNamespace(ns) - if err != nil && !errors.IsNotFound(err) { - return err - } else if errors.IsNotFound(err) { - logger.V(1).Info("creating APIServiceNamespace", "namespace", ns) - sn, err = r.createServiceNamespace(ctx, &kubebindv1alpha2.APIServiceNamespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: ns, - Namespace: r.providerNamespace, - OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(r.apiServiceExport, kubebindv1alpha2.SchemeGroupVersion.WithKind("APIServiceExport")), - }, - }, - }) - if err != nil { - return err - } - } - if sn.Status.Namespace == "" { - // note: the service provider might implement this synchronously in admission. if so, we can skip the requeue. - logger.V(1).Info("waiting for APIServiceNamespace to be ready", "namespace", ns) - return r.requeue(obj, 1*time.Second) - } - logger = logger.WithValues("upstreamNamespace", sn.Status.Namespace) - ctx = klog.NewContext(ctx, logger) + // Translate the namespace/name from the consumer side to the provider side, + // potentially creating eventually-consistent resources, hence the returned + // providerKey can be nil to indicate a requeue is desired. - // continue with upstream namespace - ns = sn.Status.Namespace + providerKey, err := r.isolationStrategy.EnsureProviderKey(ctx, types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + }) + if err != nil { + return err + } + if providerKey == nil { + return r.requeue(obj, 1*time.Second) } - upstream, err := r.getProviderObject(ns, obj.GetName()) - if err != nil && !errors.IsNotFound(err) { + upstream, err := r.getProviderObject(providerKey.Namespace, providerKey.Name) + if err != nil && !apierrors.IsNotFound(err) { return err - } else if errors.IsNotFound(err) { - if obj.GetDeletionTimestamp() != nil && !obj.GetDeletionTimestamp().IsZero() { + } else if apierrors.IsNotFound(err) { + if !obj.GetDeletionTimestamp().IsZero() { logger.V(2).Info("object is already deleting, don't sync") if _, err := r.removeDownstreamFinalizer(ctx, obj); err != nil { @@ -110,7 +93,6 @@ func (r *reconciler) reconcile(ctx context.Context, obj *unstructured.Unstructur upstream = obj.DeepCopy() upstream.SetUID("") upstream.SetResourceVersion("") - upstream.SetNamespace(ns) upstream.SetManagedFields(nil) upstream.SetDeletionTimestamp(nil) upstream.SetDeletionGracePeriodSeconds(nil) @@ -118,24 +100,35 @@ func (r *reconciler) reconcile(ctx context.Context, obj *unstructured.Unstructur upstream.SetFinalizers(nil) unstructured.RemoveNestedField(upstream.Object, "status") + // Regardless of isolation mode, we always annotate every object with its + // owning cluster namespace. + if err := r.setClusterNamespaceAnnotation(upstream); err != nil { + return err + } + + // let the isolation perform any changes it desires + if err := r.isolationStrategy.MutateMetadataAndSpec(upstream, *providerKey); err != nil { + return err + } + logger.Info("Creating upstream object") - if _, err := r.createProviderObject(ctx, upstream); err != nil && !errors.IsAlreadyExists(err) { + if _, err := r.createProviderObject(ctx, upstream); err != nil && !apierrors.IsAlreadyExists(err) { return err - } else if errors.IsAlreadyExists(err) { + } else if apierrors.IsAlreadyExists(err) { logger.Info("Upstream object already exists. Waiting for requeue.") // the upstream object will lead to a requeue } } // here the upstream already exists. Update everything but the status. - if obj.GetDeletionTimestamp() != nil && !obj.GetDeletionTimestamp().IsZero() { - if upstream.GetDeletionTimestamp() != nil && !upstream.GetDeletionTimestamp().IsZero() { + if !obj.GetDeletionTimestamp().IsZero() { + if !upstream.GetDeletionTimestamp().IsZero() { logger.V(2).Info("upstream is already deleting, wait for it") return nil // we will get an event when the upstream is deleted } logger.V(1).Info("object is already deleting downstream, deleting upstream too") - if err := r.deleteProviderObject(ctx, ns, obj.GetName()); err != nil && !errors.IsNotFound(err) { + if err := r.deleteProviderObject(ctx, providerKey.Namespace, providerKey.Name); err != nil && !apierrors.IsNotFound(err) { return err } @@ -147,6 +140,12 @@ func (r *reconciler) reconcile(ctx context.Context, obj *unstructured.Unstructur return nil // we will get an event when the upstream is deleted } + // (Re)set the annotation if it's missing, abort if for whatever reason the + // object is annotated with another cluster namespace. + if err := r.setClusterNamespaceAnnotation(upstream); err != nil { + return err + } + // just in case, checking for finalizer if obj, err = r.ensureDownstreamFinalizer(ctx, obj); err != nil { return err @@ -182,27 +181,13 @@ func (r *reconciler) reconcile(ctx context.Context, obj *unstructured.Unstructur } logger.Info("Updating upstream object") - upstream.SetManagedFields(nil) // server side apply does not want this - if _, err := r.updateProviderObject(ctx, upstream); err != nil { - return err - } - - return nil + return r.updateProviderObject(ctx, upstream) } func (r *reconciler) ensureDownstreamFinalizer(ctx context.Context, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { logger := klog.FromContext(ctx) - // check that downstream has our finalizer - found := false - for _, f := range obj.GetFinalizers() { - if f == kubebindv1alpha2.DownstreamFinalizer { - found = true - break - } - } - - if !found { + if !slices.Contains(obj.GetFinalizers(), kubebindv1alpha2.DownstreamFinalizer) { logger.V(2).Info("adding finalizer to downstream object") obj = obj.DeepCopy() obj.SetFinalizers(append(obj.GetFinalizers(), kubebindv1alpha2.DownstreamFinalizer)) @@ -240,3 +225,21 @@ func (r *reconciler) removeDownstreamFinalizer(ctx context.Context, obj *unstruc return obj, nil } + +func (r *reconciler) setClusterNamespaceAnnotation(obj *unstructured.Unstructured) error { + annnotations := obj.GetAnnotations() + existing, annotationExists := annnotations[konnectortypes.ClusterNamespaceAnnotationKey] + if annotationExists && existing != r.clusterNamespace { + return errors.New("mismatch between existing cluster namespace and given cluster namespace") + } + + if !annotationExists { + if annnotations == nil { + annnotations = map[string]string{} + } + annnotations[konnectortypes.ClusterNamespaceAnnotationKey] = r.clusterNamespace + obj.SetAnnotations(annnotations) + } + + return nil +} diff --git a/pkg/konnector/controllers/cluster/serviceexport/spec/spec_reconcile_test.go b/pkg/konnector/controllers/cluster/serviceexport/spec/spec_reconcile_test.go new file mode 100644 index 000000000..0c0847ea0 --- /dev/null +++ b/pkg/konnector/controllers/cluster/serviceexport/spec/spec_reconcile_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2023 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package spec + +import ( + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + konnectortypes "github.com/kube-bind/kube-bind/pkg/konnector/types" +) + +func TestInjectClusterNamespace(t *testing.T) { + tests := []struct { + name string + obj *unstructured.Unstructured + clusterNs string + clusterNsUID string + expected string + wantErr bool + }{ + { + name: "noExistingClusterNs", + obj: &unstructured.Unstructured{}, + clusterNs: "kube-bind-zlp9m", + clusterNsUID: "real-identity", + expected: "kube-bind-zlp9m", + wantErr: false, + }, + { + name: "oneExistingClusterNs", + obj: newObjectWithClusterNs("kube-bind-zlp9m"), + clusterNs: "kube-bind-s85lc", + clusterNsUID: "real-identity", + expected: "kube-bind-zlp9m", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + originalClusterAnn := tt.obj.GetAnnotations()[konnectortypes.ClusterNamespaceAnnotationKey] + + rec := &reconciler{ + clusterNamespace: tt.clusterNs, + } + + err := rec.setClusterNamespaceAnnotation(tt.obj) + if tt.wantErr { + require.Error(t, err) + + // ensure object was not modified + require.Equal(t, originalClusterAnn, tt.obj.GetAnnotations()[konnectortypes.ClusterNamespaceAnnotationKey]) + } else { + require.NoError(t, err) + require.Equal(t, tt.clusterNs, tt.obj.GetAnnotations()[konnectortypes.ClusterNamespaceAnnotationKey]) + } + }) + } +} + +func newObjectWithClusterNs(providerNamespace string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + ans := map[string]string{ + konnectortypes.ClusterNamespaceAnnotationKey: providerNamespace, + } + obj.SetAnnotations(ans) + ors := []metav1.OwnerReference{{ + APIVersion: "v1", + Kind: "Namespace", + Name: providerNamespace, + }} + obj.SetOwnerReferences(ors) + + return obj +} diff --git a/pkg/konnector/controllers/cluster/serviceexport/status/status_controller.go b/pkg/konnector/controllers/cluster/serviceexport/status/status_controller.go index 075e64c8a..8897623d5 100644 --- a/pkg/konnector/controllers/cluster/serviceexport/status/status_controller.go +++ b/pkg/konnector/controllers/cluster/serviceexport/status/status_controller.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" dynamicclient "k8s.io/client-go/dynamic" @@ -35,8 +36,7 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" - "github.com/kube-bind/kube-bind/pkg/indexers" - clusterscoped "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/cluster-scoped" + "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/isolation" "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/multinsinformer" "github.com/kube-bind/kube-bind/pkg/konnector/controllers/dynamic" kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" @@ -49,6 +49,7 @@ const ( // NewController returns a new controller reconciling status of upstream to downstream. func NewController( + isolationStrategy isolation.Strategy, gvr schema.GroupVersionResource, providerNamespace string, providerNamespaceUID string, @@ -94,50 +95,23 @@ func NewController( serviceNamespaceInformer: serviceNamespaceInformer, reconciler: reconciler{ - getServiceNamespace: func(upstreamNamespace string) (*kubebindv1alpha2.APIServiceNamespace, error) { - sns, err := serviceNamespaceInformer.Informer().GetIndexer().ByIndex(indexers.ServiceNamespaceByNamespace, upstreamNamespace) - if err != nil { - return nil, err - } - if len(sns) == 0 { - return nil, errors.NewNotFound(kubebindv1alpha2.SchemeGroupVersion.WithResource("APIServiceNamespace").GroupResource(), upstreamNamespace) - } - return sns[0].(*kubebindv1alpha2.APIServiceNamespace), nil - }, - getConsumerObject: func(ns, name string) (*unstructured.Unstructured, error) { + isolationStrategy: isolationStrategy, + + getConsumerObject: func(ns, name string) (obj *unstructured.Unstructured, err error) { if ns != "" { - return dynamicConsumerLister.Namespace(ns).Get(name) + obj, err = dynamicConsumerLister.Namespace(ns).Get(name) + } else { + obj, err = dynamicConsumerLister.Get(name) } - got, err := dynamicConsumerLister.Get(clusterscoped.Behead(name, providerNamespace)) - if err != nil { - return nil, err - } - obj := got.DeepCopy() - err = clusterscoped.TranslateFromDownstream(obj, providerNamespace, providerNamespaceUID) + if err != nil { return nil, err } - return obj, nil + + return obj.DeepCopy(), nil }, updateConsumerObjectStatus: func(ctx context.Context, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { - ns := obj.GetNamespace() - if ns == "" { - if err := clusterscoped.TranslateFromUpstream(obj); err != nil { - return nil, err - } - } - updated, err := consumerClient.Resource(gvr).Namespace(obj.GetNamespace()).UpdateStatus(ctx, obj, metav1.UpdateOptions{}) - if err != nil { - return nil, err - } - if ns == "" { - err = clusterscoped.TranslateFromDownstream(updated, providerNamespace, providerNamespaceUID) - if err != nil { - return nil, err - } - return updated, nil - } - return updated, nil + return consumerClient.Resource(gvr).Namespace(obj.GetNamespace()).UpdateStatus(ctx, obj, metav1.UpdateOptions{}) }, deleteProviderObject: func(ctx context.Context, ns, name string) error { return providerClient.Resource(gvr).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{}) @@ -201,34 +175,32 @@ func (c *controller) enqueueProvider(logger klog.Logger, obj any) { runtime.HandleError(err) return } - ns, _, err := cache.SplitMetaNamespaceKey(key) + + ns, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { runtime.HandleError(err) return } - if ns != "" { - sns, err := c.serviceNamespaceInformer.Informer().GetIndexer().ByIndex(indexers.ServiceNamespaceByNamespace, ns) - if err != nil { + + // Try to map the provider name back to the consumer name, + // but only to check if we "own" the object; we will actually + // enqueue the provider key after all. + consumerKey, err := c.isolationStrategy.ToConsumerKey(types.NamespacedName{ + Namespace: ns, + Name: name, + }) + if err != nil { + if !errors.IsNotFound(err) { runtime.HandleError(err) - return - } - for _, obj := range sns { - sns := obj.(*kubebindv1alpha2.APIServiceNamespace) - if sns.Namespace == c.providerNamespace { - logger.V(2).Info("queueing Unstructured", "key", key) - c.queue.Add(key) - return - } } - logger.V(3).Info("skipping because consumer mismatch", "key", key) return } - if clusterscoped.Behead(key, c.providerNamespace) == key { - logger.V(3).Info("skipping because consumer mismatch", "key", key) + if consumerKey == nil { return } - logger.V(2).Info("queueing Unstructured", "key", key) + + logger.V(2).Info("queueing Unstructured", "queued", key) c.queue.Add(key) } @@ -244,26 +216,23 @@ func (c *controller) enqueueConsumer(logger klog.Logger, obj any) { return } - if ns != "" { - sn, err := c.serviceNamespaceInformer.Lister().APIServiceNamespaces(c.providerNamespace).Get(ns) - if err != nil { - if !errors.IsNotFound(err) { - runtime.HandleError(err) - } - return - } - if sn.Namespace == c.providerNamespace && sn.Status.Namespace != "" { - key := fmt.Sprintf("%s/%s", sn.Status.Namespace, name) - logger.V(2).Info("queueing Unstructured", "key", key) - c.queue.Add(key) - return + providerKey, err := c.isolationStrategy.ToProviderKey(types.NamespacedName{ + Namespace: ns, + Name: name, + }) + if err != nil { + if !errors.IsNotFound(err) { + runtime.HandleError(err) } return } - upstreamKey := clusterscoped.Prepend(downstreamKey, c.providerNamespace) - logger.V(2).Info("queueing Unstructured", "key", upstreamKey) - c.queue.Add(upstreamKey) + if providerKey == nil { + return + } + + logger.V(2).Info("queueing Unstructured", "queued", providerKey) + c.queue.Add(providerKey.String()) } func (c *controller) enqueueServiceNamespace(logger klog.Logger, obj any) { @@ -281,15 +250,17 @@ func (c *controller) enqueueServiceNamespace(logger klog.Logger, obj any) { return // not for us } - sn, err := c.serviceNamespaceInformer.Lister().APIServiceNamespaces(ns).Get(name) + strategy := c.isolationStrategy.(*isolation.ServiceNamespacedStrategy) + nsOnProviderCluster, err := strategy.ProviderNamespace(name) if err != nil { runtime.HandleError(err) return } - if sn.Status.Namespace == "" { + if nsOnProviderCluster == "" { return // not ready } - objs, err := c.providerDynamicInformer.List(sn.Status.Namespace) + + objs, err := c.providerDynamicInformer.List(nsOnProviderCluster) if err != nil { runtime.HandleError(err) return @@ -300,7 +271,7 @@ func (c *controller) enqueueServiceNamespace(logger klog.Logger, obj any) { runtime.HandleError(err) continue } - logger.V(2).Info("queueing Unstructured", "key", key, "reason", "APIServiceNamespace", "ServiceNamespaceKey", snKey) + logger.V(2).Info("queueing Unstructured", "queued", key, "reason", "APIServiceNamespace", "ServiceNamespaceKey", snKey) c.queue.Add(key) } } @@ -315,17 +286,22 @@ func (c *controller) Start(ctx context.Context, numThreads int) { logger.Info("Starting controller") defer logger.Info("Shutting down controller") - c.serviceNamespaceInformer.Informer().AddDynamicEventHandler(ctx, controllerName, cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj any) { - c.enqueueServiceNamespace(logger, obj) - }, - UpdateFunc: func(_, newObj any) { - c.enqueueServiceNamespace(logger, newObj) - }, - DeleteFunc: func(obj any) { - c.enqueueServiceNamespace(logger, obj) - }, - }) + // APIServiceNamespaces are only of interest when syncing namespaced + // objects, and since these event handlers need the appropriate isolation + // strategy, we only start them when necessary. + if _, ok := c.isolationStrategy.(*isolation.ServiceNamespacedStrategy); ok { + c.serviceNamespaceInformer.Informer().AddDynamicEventHandler(ctx, controllerName, cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { + c.enqueueServiceNamespace(logger, obj) + }, + UpdateFunc: func(_, newObj any) { + c.enqueueServiceNamespace(logger, newObj) + }, + DeleteFunc: func(obj any) { + c.enqueueServiceNamespace(logger, obj) + }, + }) + } for i := 0; i < numThreads; i++ { go wait.UntilWithContext(ctx, c.startWorker, time.Second) diff --git a/pkg/konnector/controllers/cluster/serviceexport/status/status_reconcile.go b/pkg/konnector/controllers/cluster/serviceexport/status/status_reconcile.go index 216e5d048..69c7d44ff 100644 --- a/pkg/konnector/controllers/cluster/serviceexport/status/status_reconcile.go +++ b/pkg/konnector/controllers/cluster/serviceexport/status/status_reconcile.go @@ -22,14 +22,15 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/klog/v2" - kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" + "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/isolation" ) type reconciler struct { - getServiceNamespace func(upstreamNamespace string) (*kubebindv1alpha2.APIServiceNamespace, error) + isolationStrategy isolation.Strategy getConsumerObject func(ns, name string) (*unstructured.Unstructured, error) updateConsumerObjectStatus func(ctx context.Context, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) @@ -41,37 +42,43 @@ type reconciler struct { func (r *reconciler) reconcile(ctx context.Context, obj *unstructured.Unstructured) error { logger := klog.FromContext(ctx) - ns := obj.GetNamespace() - if ns != "" { - sn, err := r.getServiceNamespace(ns) - if err != nil && !errors.IsNotFound(err) { + // Map the provider object namespace/name to the consumer side. Technically + // consumerKey should never be nil (for it to be nil, the namespace in the + // APIServiceNamespace's status must be blank, but it's blank, how could we + // have found an object in a non-defined namespace)?, but we are on the safe + // side and handle it anyway. + consumerKey, err := r.isolationStrategy.ToConsumerKey(types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + }) + if err != nil { + if !errors.IsNotFound(err) { return err - } else if errors.IsNotFound(err) { - runtime.HandleError(err) - return err // hoping the APIServiceNamespace will be created soon. Otherwise, this item goes into backoff. - } - if sn.Status.Namespace == "" { - runtime.HandleError(err) - return err // hoping the status is set soon. } - logger = logger.WithValues("upstreamNamespace", sn.Status.Namespace) - ctx = klog.NewContext(ctx, logger) - - // continue with downstream namespace - ns = sn.Name + return nil + } + if consumerKey == nil { + return nil } - downstream, err := r.getConsumerObject(ns, obj.GetName()) + logger = logger.WithValues("downstream", consumerKey) + + downstream, err := r.getConsumerObject(consumerKey.Namespace, consumerKey.Name) if err != nil { if errors.IsNotFound(err) { // downstream is gone. Delete upstream too. Note that we cannot rely on the spec controller because // due to konnector restart it might have missed the deletion event. - logger.Info("Deleting upstream object because downstream is gone", "downstreamNamespace", ns, "downstreamName", obj.GetName()) + logger.Info("Deleting upstream object because downstream is gone") return r.deleteProviderObject(ctx, obj.GetNamespace(), obj.GetName()) } - logger.Info("failed to get downstream object", "error", err, "downstreamNamespace", ns, "downstreamName", obj.GetName()) + logger.Error(err, "failed to get downstream object") + return err + } + + // let the isolation perform any changes it desires + if err := r.isolationStrategy.MutateStatus(downstream, *consumerKey); err != nil { return err } @@ -90,6 +97,7 @@ func (r *reconciler) reconcile(ctx context.Context, obj *unstructured.Unstructur } else { unstructured.RemoveNestedField(downstream.Object, "status") } + if !reflect.DeepEqual(orig, downstream) { logger.Info("Updating downstream object status") if _, err := r.updateConsumerObjectStatus(ctx, downstream); err != nil { diff --git a/pkg/konnector/types/types.go b/pkg/konnector/types/types.go new file mode 100644 index 000000000..9adce955c --- /dev/null +++ b/pkg/konnector/types/types.go @@ -0,0 +1,23 @@ +/* +Copyright 2025 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +// ClusterNamespaceAnnotationKey is the annotation key to identify the +// cluster namespace that any synced object on the provider side belongs to. +// This annotation is set on all synced objects, regardless of scope or +// isolation mode. +const ClusterNamespaceAnnotationKey = "kube-bind.io/cluster-namespace" diff --git a/test/e2e/bind/happy-case_test.go b/test/e2e/bind/happy-case_test.go index d30c0b9b5..12a8791fe 100644 --- a/test/e2e/bind/happy-case_test.go +++ b/test/e2e/bind/happy-case_test.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/require" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -33,13 +34,14 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" "k8s.io/client-go/util/retry" "sigs.k8s.io/yaml" kuberesources "github.com/kube-bind/kube-bind/backend/kubernetes/resources" bindapiservice "github.com/kube-bind/kube-bind/cli/pkg/kubectl/bind-apiservice/plugin" examples "github.com/kube-bind/kube-bind/deploy/examples" - clusterscoped "github.com/kube-bind/kube-bind/pkg/konnector/controllers/cluster/serviceexport/cluster-scoped" + "github.com/kube-bind/kube-bind/pkg/konnector/types" kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" "github.com/kube-bind/kube-bind/test/e2e/framework" ) @@ -47,21 +49,25 @@ import ( func TestClusterScoped(t *testing.T) { t.Parallel() // name & test type defined by letters - cc - cluster-cluster so its easier to identify failures in the logs. - testHappyCase(t, "cc", apiextensionsv1.ClusterScoped, kubebindv1alpha2.ClusterScope) + testHappyCase(t, "cc-prefixed", apiextensionsv1.ClusterScoped, apiextensionsv1.ClusterScoped, kubebindv1alpha2.ClusterScope, kubebindv1alpha2.IsolationPrefixed) + testHappyCase(t, "cc-none", apiextensionsv1.ClusterScoped, apiextensionsv1.ClusterScoped, kubebindv1alpha2.ClusterScope, kubebindv1alpha2.IsolationNone) + testHappyCase(t, "cc-namespaced", apiextensionsv1.ClusterScoped, apiextensionsv1.NamespaceScoped, kubebindv1alpha2.ClusterScope, kubebindv1alpha2.IsolationNamespaced) } func TestNamespacedScoped(t *testing.T) { t.Parallel() - testHappyCase(t, "nn", apiextensionsv1.NamespaceScoped, kubebindv1alpha2.NamespacedScope) - testHappyCase(t, "nc", apiextensionsv1.NamespaceScoped, kubebindv1alpha2.ClusterScope) + testHappyCase(t, "nn", apiextensionsv1.NamespaceScoped, apiextensionsv1.NamespaceScoped, kubebindv1alpha2.NamespacedScope, "") + testHappyCase(t, "nc", apiextensionsv1.NamespaceScoped, apiextensionsv1.NamespaceScoped, kubebindv1alpha2.ClusterScope, "") } func testHappyCase( t *testing.T, name string, - resourceScope apiextensionsv1.ResourceScope, + consumerResourceScope apiextensionsv1.ResourceScope, + providerResourceScope apiextensionsv1.ResourceScope, informerScope kubebindv1alpha2.InformerScope, + isolationStrategy kubebindv1alpha2.Isolation, ) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) // Commented out to prevent cleanup of kcp assets @@ -72,24 +78,36 @@ func testHappyCase( providerConfig, providerKubeconfig := framework.NewWorkspace(t, framework.ClientConfig(t), framework.WithName("%s-provider-%s", name, suffix)) t.Logf("Installing kubebind CRDs") - framework.InstallKubebindCRDs(t, providerConfig) + framework.InstallKubeBindCRDs(t, providerConfig) t.Logf("Starting backend with random port") - addr, _ := framework.StartBackend(t, "--kubeconfig="+providerKubeconfig, "--listen-address=:0", "--consumer-scope="+string(informerScope)) + addr, _ := framework.StartBackend(t, + "--kubeconfig="+providerKubeconfig, + "--listen-address=:0", + "--consumer-scope="+string(informerScope), + "--cluster-scoped-isolation="+string(isolationStrategy), + ) t.Logf("Creating CRD on provider side") examples.Bootstrap(t, framework.DiscoveryClient(t, providerConfig), framework.DynamicClient(t, providerConfig), nil) + // For namespaced-isolated cluster-scoped objects, the CRD needs to be namespaced on the provider side, + // but cluster-scoped on the consumer side. To make this setup possible without introducing another CRD, + // we will simply "hack" the CRD here on the providerside to make it work. + if providerResourceScope == apiextensionsv1.NamespaceScoped && consumerResourceScope == apiextensionsv1.ClusterScoped { + t.Logf("Changing sheriff CRD scope to namespaced on the provider side") + toggleCRDScope(t, ctx, providerConfig, "sheriffs.wildwest.dev", apiextensionsv1.NamespaceScoped) + } + t.Logf("Creating consumer workspace and starting konnector") consumerConfig, consumerKubeconfig := framework.NewWorkspace(t, framework.ClientConfig(t), framework.WithName("%s-consumer-%s", name, suffix)) framework.StartKonnector(t, consumerConfig, "--kubeconfig="+consumerKubeconfig) serviceGVR := schema.GroupVersionResource{Group: "wildwest.dev", Version: "v1alpha1", Resource: "cowboys"} - if resourceScope == apiextensionsv1.ClusterScoped { - serviceGVR = schema.GroupVersionResource{Group: "wildwest.dev", Version: "v1alpha1", Resource: "sheriffs"} - } templateRef := "cowboys" - if resourceScope == apiextensionsv1.ClusterScoped { + + if consumerResourceScope == apiextensionsv1.ClusterScoped { + serviceGVR = schema.GroupVersionResource{Group: "wildwest.dev", Version: "v1alpha1", Resource: "sheriffs"} templateRef = "sheriffs" } @@ -100,7 +118,16 @@ func testHappyCase( providerBindClient := framework.BindClient(t, providerConfig) // Instance variables removed - now seeded directly in test - consumerNS, providerNS := "wild-west", "unknown" + // These two namespaces are where the "main" object resides on each cluster. + consumerNs, providerNs := "wild-west", "unknown" + + // When namespaced isolation is used (i.e. cluster-scoped objects on the consumer + // turn into namespaced objects on the provider cluster), permission claimed objects + // are still synced into their respective APIServiceNamespace-managed namespaces. + permClaimNs := "unknown" + + // cluster namespace is the main "contract" namespace, i.e. where the BoundSchema and other + // bind-related objects reside. clusterNs, clusterScopedUpInsName := "unknown", "unknown" kubeBindConfig := path.Join(framework.WorkDir, "kube-bind-config.yaml") @@ -110,7 +137,7 @@ func testHappyCase( // For sheriffs: sheriff-badge-credentials (referenced), sheriff-jurisdiction-config (label selector) var referencedSecretName, labelSelectedSecretName string var filename string - if resourceScope == apiextensionsv1.NamespaceScoped { + if consumerResourceScope == apiextensionsv1.NamespaceScoped { referencedSecretName = "colt-45-permit" //nolint:gosec labelSelectedSecretName = "cowboy-gang-affiliation" filename = "cr-cowboy.yaml" @@ -218,10 +245,18 @@ func testHappyCase( }, wait.ForeverTestTimeout, time.Millisecond*100, "waiting for the %s instance to be created on provider side", serviceGVR.Resource) // these are used everywhere further down - providerNS = instances.Items[0].GetNamespace() + firstObj := instances.Items[0] + + // providerNs is the namespaced where the synced object lives; this might be empty + // for cluster-scoped objects on the provider side. + providerNs = firstObj.GetNamespace() + // Cluster namespace represent binding contract namespace, and is stored on every object. - clusterNs, _ = clusterscoped.ExtractClusterNs(&instances.Items[0]) - clusterScopedUpInsName = clusterscoped.Prepend("test", clusterNs) + clusterNs = firstObj.GetAnnotations()[types.ClusterNamespaceAnnotationKey] + require.NotEmpty(t, clusterNs, "cluster namespace annotation must always exist") + + // the object name on the provider side; this is only used for cluster-scoped objects + clusterScopedUpInsName = firstObj.GetName() }, }, // Request included namespace, so we check it first @@ -232,8 +267,8 @@ func testHappyCase( var foundPreSeededNamespace bool var actualProviderNamespace string require.Eventually(t, func() bool { - // If are operating namespaced resources - namespace will be set, if cluster - we need to use - // extraced one from the cluster-scoped object. + // If we are operating namespaced resources - namespace will be set, if cluster - we need to use + // extracted one from the cluster-scoped object. namespaces, err := providerBindClient.KubeBindV1alpha2().APIServiceNamespaces(clusterNs).List(ctx, metav1.ListOptions{}) if err != nil { return false @@ -300,7 +335,7 @@ func testHappyCase( t.Logf("Verifying RBAC resources were created for secret management in namespace scope") rbacClient := framework.KubeClient(t, providerConfig).RbacV1() - roles, err := rbacClient.Roles(providerNS).List(ctx, metav1.ListOptions{}) + roles, err := rbacClient.Roles(providerNs).List(ctx, metav1.ListOptions{}) require.NoError(t, err) var foundSecretRole bool @@ -321,7 +356,7 @@ func testHappyCase( require.True(t, foundSecretRole, "Role for secrets should be created") t.Logf("Verifying RoleBinding was created for pre-seeded namespace secret access") - roleBindings, err := rbacClient.RoleBindings(providerNS).List(ctx, metav1.ListOptions{}) + roleBindings, err := rbacClient.RoleBindings(providerNs).List(ctx, metav1.ListOptions{}) require.NoError(t, err) var foundSecretRoleBinding bool @@ -357,8 +392,8 @@ func testHappyCase( // We need to establish namespace only in cluster scope for cluster scoped resources. // Else we can trust sync object namespace as it will be the same. if informerScope == kubebindv1alpha2.ClusterScope && - resourceScope == apiextensionsv1.ClusterScoped { - if providerNS == "unknown" { + consumerResourceScope == apiextensionsv1.ClusterScoped { + if providerNs == "unknown" { t.Fatal("providerNS is not set. Programming error in the test.") } @@ -371,15 +406,19 @@ func testHappyCase( } for _, namespace := range namespaces.Items { - if strings.Contains(namespace.Name, consumerNS) && namespace.Status.Namespace != "" { - providerNS = namespace.Status.Namespace + if strings.Contains(namespace.Name, consumerNs) && namespace.Status.Namespace != "" { + permClaimNs = namespace.Status.Namespace return true } } return false }, wait.ForeverTestTimeout, time.Millisecond*100, "waiting for APIServiceNamespace to be created on provider side") - require.NotEmpty(t, providerNS, "No cluster namespaces found") + require.NotEmpty(t, permClaimNs, "No permission claim namespaces found") + + t.Logf("permclaim namespace detected as: %q", permClaimNs) + } else { + permClaimNs = providerNs } }, }, @@ -388,13 +427,13 @@ func testHappyCase( step: func(t *testing.T) { t.Logf("Waiting for referenced secret to be synced to provider side") require.Eventually(t, func() bool { - _, err := providerCoreClient.Secrets(providerNS).Get(ctx, referencedSecretName, metav1.GetOptions{}) + _, err := providerCoreClient.Secrets(permClaimNs).Get(ctx, referencedSecretName, metav1.GetOptions{}) return err == nil }, time.Minute*2, time.Millisecond*100, "waiting for referenced secret to be synced to provider side") t.Logf("Waiting for label-selected secret to be synced to provider side") require.Eventually(t, func() bool { - _, err := providerCoreClient.Secrets(providerNS).Get(ctx, labelSelectedSecretName, metav1.GetOptions{}) + _, err := providerCoreClient.Secrets(permClaimNs).Get(ctx, labelSelectedSecretName, metav1.GetOptions{}) return err == nil }, wait.ForeverTestTimeout, time.Millisecond*100, "waiting for label-selected secret to be synced to provider side") @@ -413,8 +452,8 @@ func testHappyCase( name: "instance deleted upstream is recreated", step: func(t *testing.T) { var err error - if resourceScope == apiextensionsv1.NamespaceScoped { - err = providerClient.Namespace(providerNS).Delete(ctx, "test", metav1.DeleteOptions{}) + if providerResourceScope == apiextensionsv1.NamespaceScoped { + err = providerClient.Namespace(providerNs).Delete(ctx, "test", metav1.DeleteOptions{}) } else { err = providerClient.Delete(ctx, clusterScopedUpInsName, metav1.DeleteOptions{}) } @@ -422,8 +461,8 @@ func testHappyCase( require.Eventually(t, func() bool { var err error - if resourceScope == apiextensionsv1.NamespaceScoped { - _, err = providerClient.Namespace(providerNS).Get(ctx, "test", metav1.GetOptions{}) + if providerResourceScope == apiextensionsv1.NamespaceScoped { + _, err = providerClient.Namespace(providerNs).Get(ctx, "test", metav1.GetOptions{}) } else { _, err = providerClient.Get(ctx, clusterScopedUpInsName, metav1.GetOptions{}) } @@ -437,15 +476,15 @@ func testHappyCase( err := retry.RetryOnConflict(retry.DefaultRetry, func() error { var obj *unstructured.Unstructured var err error - if resourceScope == apiextensionsv1.NamespaceScoped { - obj, err = consumerClient.Namespace(consumerNS).Get(ctx, "test", metav1.GetOptions{}) + if consumerResourceScope == apiextensionsv1.NamespaceScoped { + obj, err = consumerClient.Namespace(consumerNs).Get(ctx, "test", metav1.GetOptions{}) } else { obj, err = consumerClient.Get(ctx, "test", metav1.GetOptions{}) } require.NoError(t, err) - if resourceScope == apiextensionsv1.NamespaceScoped { + if consumerResourceScope == apiextensionsv1.NamespaceScoped { unstructured.SetNestedField(obj.Object, "Updated cowboy intent", "spec", "intent") //nolint:errcheck - _, err = consumerClient.Namespace(consumerNS).Update(ctx, obj, metav1.UpdateOptions{}) + _, err = consumerClient.Namespace(consumerNs).Update(ctx, obj, metav1.UpdateOptions{}) } else { unstructured.SetNestedField(obj.Object, "Updated sheriff intent", "spec", "intent") //nolint:errcheck _, err = consumerClient.Update(ctx, obj, metav1.UpdateOptions{}) @@ -457,8 +496,8 @@ func testHappyCase( require.Eventually(t, func() bool { var obj *unstructured.Unstructured var err error - if resourceScope == apiextensionsv1.NamespaceScoped { - obj, err = providerClient.Namespace(providerNS).Get(ctx, "test", metav1.GetOptions{}) + if providerResourceScope == apiextensionsv1.NamespaceScoped { + obj, err = providerClient.Namespace(providerNs).Get(ctx, "test", metav1.GetOptions{}) } else { obj, err = providerClient.Get(ctx, clusterScopedUpInsName, metav1.GetOptions{}) } @@ -467,7 +506,7 @@ func testHappyCase( value, _, err = unstructured.NestedString(obj.Object, "spec", "intent") require.NoError(t, err) - if resourceScope == apiextensionsv1.NamespaceScoped { + if consumerResourceScope == apiextensionsv1.NamespaceScoped { return value == "Updated cowboy intent" } else { return value == "Updated sheriff intent" @@ -481,15 +520,15 @@ func testHappyCase( err := retry.RetryOnConflict(retry.DefaultRetry, func() error { var obj *unstructured.Unstructured var err error - if resourceScope == apiextensionsv1.NamespaceScoped { - obj, err = providerClient.Namespace(providerNS).Get(ctx, "test", metav1.GetOptions{}) + if providerResourceScope == apiextensionsv1.NamespaceScoped { + obj, err = providerClient.Namespace(providerNs).Get(ctx, "test", metav1.GetOptions{}) } else { obj, err = providerClient.Get(ctx, clusterScopedUpInsName, metav1.GetOptions{}) } require.NoError(t, err) unstructured.SetNestedField(obj.Object, "Ready to ride", "status", "result") //nolint:errcheck - if resourceScope == apiextensionsv1.NamespaceScoped { - _, err = providerClient.Namespace(providerNS).UpdateStatus(ctx, obj, metav1.UpdateOptions{}) + if providerResourceScope == apiextensionsv1.NamespaceScoped { + _, err = providerClient.Namespace(providerNs).UpdateStatus(ctx, obj, metav1.UpdateOptions{}) } else { _, err = providerClient.UpdateStatus(ctx, obj, metav1.UpdateOptions{}) } @@ -500,8 +539,8 @@ func testHappyCase( require.Eventually(t, func() bool { var obj *unstructured.Unstructured var err error - if resourceScope == apiextensionsv1.NamespaceScoped { - obj, err = consumerClient.Namespace(consumerNS).Get(ctx, "test", metav1.GetOptions{}) + if consumerResourceScope == apiextensionsv1.NamespaceScoped { + obj, err = consumerClient.Namespace(consumerNs).Get(ctx, "test", metav1.GetOptions{}) } else { obj, err = consumerClient.Get(ctx, "test", metav1.GetOptions{}) } @@ -518,15 +557,15 @@ func testHappyCase( err := retry.RetryOnConflict(retry.DefaultRetry, func() error { var obj *unstructured.Unstructured var err error - if resourceScope == apiextensionsv1.NamespaceScoped { - obj, err = providerClient.Namespace(providerNS).Get(ctx, "test", metav1.GetOptions{}) + if providerResourceScope == apiextensionsv1.NamespaceScoped { + obj, err = providerClient.Namespace(providerNs).Get(ctx, "test", metav1.GetOptions{}) } else { obj, err = providerClient.Get(ctx, clusterScopedUpInsName, metav1.GetOptions{}) } require.NoError(t, err) - if resourceScope == apiextensionsv1.NamespaceScoped { + if providerResourceScope == apiextensionsv1.NamespaceScoped { unstructured.SetNestedField(obj.Object, "Drifted cowboy intent", "spec", "intent") //nolint:errcheck - _, err = providerClient.Namespace(providerNS).Update(ctx, obj, metav1.UpdateOptions{}) + _, err = providerClient.Namespace(providerNs).Update(ctx, obj, metav1.UpdateOptions{}) } else { unstructured.SetNestedField(obj.Object, "Drifted sheriff intent", "spec", "intent") //nolint:errcheck _, err = providerClient.Update(ctx, obj, metav1.UpdateOptions{}) @@ -538,14 +577,14 @@ func testHappyCase( require.Eventually(t, func() bool { var obj *unstructured.Unstructured var err error - if resourceScope == apiextensionsv1.NamespaceScoped { - obj, err = providerClient.Namespace(providerNS).Get(ctx, "test", metav1.GetOptions{}) + if providerResourceScope == apiextensionsv1.NamespaceScoped { + obj, err = providerClient.Namespace(providerNs).Get(ctx, "test", metav1.GetOptions{}) } else { obj, err = providerClient.Get(ctx, clusterScopedUpInsName, metav1.GetOptions{}) } require.NoError(t, err) var value string - if resourceScope == apiextensionsv1.NamespaceScoped { + if consumerResourceScope == apiextensionsv1.NamespaceScoped { value, _, err = unstructured.NestedString(obj.Object, "spec", "intent") require.NoError(t, err) return value == "Updated cowboy intent" @@ -561,8 +600,8 @@ func testHappyCase( name: "instances deleted downstream are deleted upstream", step: func(t *testing.T) { var err error - if resourceScope == apiextensionsv1.NamespaceScoped { - err = consumerClient.Namespace(consumerNS).Delete(ctx, "test", metav1.DeleteOptions{}) + if consumerResourceScope == apiextensionsv1.NamespaceScoped { + err = consumerClient.Namespace(consumerNs).Delete(ctx, "test", metav1.DeleteOptions{}) } else { err = consumerClient.Delete(ctx, "test", metav1.DeleteOptions{}) } @@ -570,8 +609,8 @@ func testHappyCase( require.Eventually(t, func() bool { var err error - if resourceScope == apiextensionsv1.NamespaceScoped { - _, err = providerClient.Namespace(providerNS).Get(ctx, "test", metav1.GetOptions{}) + if providerResourceScope == apiextensionsv1.NamespaceScoped { + _, err = providerClient.Namespace(providerNs).Get(ctx, "test", metav1.GetOptions{}) } else { _, err = providerClient.Get(ctx, clusterScopedUpInsName, metav1.GetOptions{}) } @@ -666,3 +705,30 @@ func applyMultiDocYAML(ctx context.Context, t *testing.T, dynamicClient dynamic. return nil } + +func toggleCRDScope(t *testing.T, ctx context.Context, config *rest.Config, name string, scope apiextensionsv1.ResourceScope) { + clientset, err := apiextensionsclient.NewForConfig(config) + require.NoError(t, err) + + crdClient := clientset.ApiextensionsV1().CustomResourceDefinitions() + + // copy existing CRD + crd, err := crdClient.Get(ctx, name, metav1.GetOptions{}) + require.NoError(t, err) + + // delete it + require.NoError(t, crdClient.Delete(ctx, name, metav1.DeleteOptions{})) + + require.Eventually(t, func() bool { + _, err := crdClient.Get(ctx, name, metav1.GetOptions{}) + return errors.IsNotFound(err) + }, wait.ForeverTestTimeout, time.Millisecond*100, "waiting for the CRD to be deleted") + + // re-create it + crd.Spec.Scope = scope + crd.ObjectMeta.ResourceVersion = "" + crd.ObjectMeta.UID = "" + crd.ObjectMeta.Generation = 0 + _, err = crdClient.Create(ctx, crd, metav1.CreateOptions{}) + require.NoError(t, err) +} diff --git a/test/e2e/framework/backend.go b/test/e2e/framework/backend.go index 843ebc356..a98e94ec4 100644 --- a/test/e2e/framework/backend.go +++ b/test/e2e/framework/backend.go @@ -34,7 +34,7 @@ import ( kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" ) -func InstallKubebindCRDs(t testing.TB, clientConfig *rest.Config) { +func InstallKubeBindCRDs(t testing.TB, clientConfig *rest.Config) { crdClient, err := apiextensionsclient.NewForConfig(clientConfig) require.NoError(t, err) err = crd.Create(t.Context(), diff --git a/test/e2e/framework/dex.go b/test/e2e/framework/dex.go index a5a32e3e4..e09919461 100644 --- a/test/e2e/framework/dex.go +++ b/test/e2e/framework/dex.go @@ -72,7 +72,7 @@ func StartDex(t testing.TB) { dexConfig, ) - // Set os-dependend killing + // Set os-dependent killing dexKill(t, dexCmd) require.NoError(t, dexCmd.Start())