diff --git a/Dockerfile.buildx b/Dockerfile.buildx index b06039d2..d1f9f88c 100644 --- a/Dockerfile.buildx +++ b/Dockerfile.buildx @@ -1,10 +1,9 @@ -FROM --platform=$BUILDPLATFORM golang:1.18 AS deps +FROM --platform=$BUILDPLATFORM golang:1.22 AS build ARG TARGETPLATFORM ARG BUILDPLATFORM COPY . /src WORKDIR /src -RUN go get -v ./... RUN go vet -v ./... RUN CGO_ENABLED=0 GO111MODULE=on go build @@ -12,6 +11,6 @@ FROM --platform=$TARGETPLATFORM scratch ARG TARGETPLATFORM LABEL MAINTAINER="Martin Helmich " -COPY --from=build /src/ /kubernetes-replicator +COPY --from=build /src/kubernetes-replicator /replicator -CMD ["/kubernetes-replicator"] +CMD ["/replicator"] diff --git a/README.md b/README.md index a07bb66d..ff2500b6 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ secrets and config maps available in multiple namespaces. 1. [1. Create the source secret](#step-1-create-the-source-secret) 1. [2. Create empty secret](#step-2-create-an-empty-destination-secret) 1. [Special case: TLS secrets](#special-case-tls-secrets) + 1. [Special case: Service replication](#special-case-service-replication) +1. [Local development/testing](#local-developmenttesting-with-minikube) ## Deployment @@ -111,6 +113,11 @@ When the labels of a namespace are changed, any resources that were replicated b It is possible to use both methods of push-based replication together in a single resource, by specifying both annotations. +#### :warning: "push-based" is dangerous +:warning: "push-based" setup is dangerous as it allows an actor to influence (read: overwrite) sensitive resources in a cluster. + +Please consider to only enable the features you actually need - see `values.yaml:replicationEnabled[]`. + ### "Pull-based" replication Pull-based replication makes it possible to create a secret/configmap/role/rolebindings and select a "source" resource @@ -241,3 +248,69 @@ data: ``` See also: https://github.com/mittwald/kubernetes-replicator/issues/120 + +### Special case: Service replication + +An annotated `kind: Service` will be replicated to another namespace as `type: ExternalName`. This feature allows to cover 2 use cases + +1) common DNS domain for services or a very lightweight service mesh +2) migration of services into own namespaces while keeping their known DNS names + + +This service + ```yaml +apiVersion: v1 +kind: Service +metadata: + name: source-service + namespace: default + annotations: + alb.ingress.kubernetes.io/backend-protocol: HTTP + alb.ingress.kubernetes.io/healthcheck-path: /version + alb.ingress.kubernetes.io/healthcheck-protocol: HTTP + replicator.v1.mittwald.de/replicate-to: some-namespace +spec: + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP + selector: + app: foo + application: foo +``` + +will be replicated to this +```yaml +apiVersion: v1 +kind: Service +metadata: + name: source-service + namespace: some-namespace + annotations: + replicator.v1.mittwald.de/replicated-at: "2024-08-21T09:07:45Z" + replicator.v1.mittwald.de/replicated-from-version: "680" +spec: + type: ExternalName + externalName: source-service.default.svc.cluster.local. + sessionAffinity: None +``` + +Please note: +- `metadata.annotations` are **not replicated** by default as on a `kind: service` they usually drive load-balancer operators. You can explicitly set `replicator.v1.mittwald.de/strip-annotations: "false"` to keep them. +- there is only the `replicator.v1.mittwald.de/replicate-to` option implemented +- pre-existing target `kind: Service` will happily be patched/overwritten ;-) + +## Local development/testing with minikube +- start a minikube cluster + - `minikube start --kubernetes-version=latest` +- build the image (adjust your platform) + - `minikube image build -t quay.io/mittwald/kubernetes-replicator:latest -f Dockerfile.buildx --build-env=TARGETPLATFORM=linux/amd64 --build-env=BUILDPLATFORM=linux/amd64 .` + - `minikube image ls` +- deploy replicator + - `kubectl apply -f deploy/rbac.yaml` + - `kubectl apply -f deploy/deployment.yaml` + - `kubectl get pods -n kube-system` +- deploy test sources + - `kubectl apply --kustomize test` +- happy replication! diff --git a/config.go b/config.go index 2ce0cf35..5672f094 100644 --- a/config.go +++ b/config.go @@ -14,5 +14,6 @@ type flags struct { ReplicateConfigMaps bool ReplicateRoles bool ReplicateRoleBindings bool + ReplicateServices bool ReplicateServiceAccounts bool } diff --git a/deploy/deployment.yaml b/deploy/deployment.yaml index 8af8c518..26903102 100644 --- a/deploy/deployment.yaml +++ b/deploy/deployment.yaml @@ -24,7 +24,7 @@ spec: - name: kubernetes-replicator securityContext: {} image: quay.io/mittwald/kubernetes-replicator:latest - imagePullPolicy: Always + imagePullPolicy: IfNotPresent args: [] ports: - name: health diff --git a/deploy/helm-chart/kubernetes-replicator/templates/deployment.yaml b/deploy/helm-chart/kubernetes-replicator/templates/deployment.yaml index 462bfe95..81fbf780 100644 --- a/deploy/helm-chart/kubernetes-replicator/templates/deployment.yaml +++ b/deploy/helm-chart/kubernetes-replicator/templates/deployment.yaml @@ -48,6 +48,7 @@ spec: - -replicate-roles={{ .Values.replicationEnabled.roles }} - -replicate-role-bindings={{ .Values.replicationEnabled.roleBindings }} - -replicate-service-accounts={{ .Values.replicationEnabled.serviceAccounts }} + - -replicate-services={{ .Values.replicationEnabled.services }} {{- with .Values.args }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/deploy/helm-chart/kubernetes-replicator/templates/rbac.yaml b/deploy/helm-chart/kubernetes-replicator/templates/rbac.yaml index 920b513b..4983b24a 100644 --- a/deploy/helm-chart/kubernetes-replicator/templates/rbac.yaml +++ b/deploy/helm-chart/kubernetes-replicator/templates/rbac.yaml @@ -27,7 +27,7 @@ rules: - watch - list {{ with .Values.replicationEnabled }} -{{- if or .secrets .configMaps .serviceAccounts }} +{{- if or .secrets .configMaps .serviceAccounts .services }} - apiGroups: - "" resources: @@ -39,6 +39,9 @@ rules: {{- end }} {{- if .serviceAccounts }} - serviceaccounts +{{- end }} +{{- if .services }} + - services {{- end }} verbs: - get diff --git a/deploy/helm-chart/kubernetes-replicator/values.yaml b/deploy/helm-chart/kubernetes-replicator/values.yaml index 45a702ce..fade66fd 100644 --- a/deploy/helm-chart/kubernetes-replicator/values.yaml +++ b/deploy/helm-chart/kubernetes-replicator/values.yaml @@ -1,7 +1,7 @@ image: repository: quay.io/mittwald/kubernetes-replicator #tag: stable # if no tag is given, the chart's appVersion is used - pullPolicy: Always + pullPolicy: IfNotPresent imagePullSecrets: [] nameOverride: "" fullnameOverride: "" @@ -16,6 +16,7 @@ replicationEnabled: roles: true roleBindings: true serviceAccounts: true + services: true ## Deployment strategy / DaemonSet updateStrategy ## diff --git a/deploy/rbac.yaml b/deploy/rbac.yaml index 5b0caa45..390b68e0 100644 --- a/deploy/rbac.yaml +++ b/deploy/rbac.yaml @@ -13,7 +13,7 @@ rules: resources: [ "namespaces" ] verbs: [ "get", "watch", "list" ] - apiGroups: [""] # "" indicates the core API group - resources: ["secrets", "configmaps", "serviceaccounts"] + resources: ["secrets", "configmaps", "serviceaccounts", "services"] verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] diff --git a/main.go b/main.go index e1e3cea7..c463f17f 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "github.com/mittwald/kubernetes-replicator/replicate/role" "github.com/mittwald/kubernetes-replicator/replicate/rolebinding" "github.com/mittwald/kubernetes-replicator/replicate/secret" + "github.com/mittwald/kubernetes-replicator/replicate/service" "github.com/mittwald/kubernetes-replicator/replicate/serviceaccount" log "github.com/sirupsen/logrus" @@ -36,6 +37,7 @@ func init() { flag.BoolVar(&f.ReplicateRoles, "replicate-roles", true, "Enable replication of roles") flag.BoolVar(&f.ReplicateRoleBindings, "replicate-role-bindings", true, "Enable replication of role bindings") flag.BoolVar(&f.ReplicateServiceAccounts, "replicate-service-accounts", true, "Enable replication of service accounts") + flag.BoolVar(&f.ReplicateServices, "replicate-services", true, "Enable replication of services") flag.Parse() switch strings.ToUpper(strings.TrimSpace(f.LogLevel)) { @@ -117,6 +119,12 @@ func main() { enabledReplicators = append(enabledReplicators, serviceAccountRepl) } + if f.ReplicateServices { + serviceRepl := service.NewReplicator(client, f.ResyncPeriod, f.AllowAll) + go serviceRepl.Run() + enabledReplicators = append(enabledReplicators, serviceRepl) + } + h := liveness.Handler{ Replicators: enabledReplicators, } diff --git a/replicate/common/consts.go b/replicate/common/consts.go index db18cb38..98b51fc1 100644 --- a/replicate/common/consts.go +++ b/replicate/common/consts.go @@ -12,4 +12,5 @@ const ( ReplicateToMatching = "replicator.v1.mittwald.de/replicate-to-matching" KeepOwnerReferences = "replicator.v1.mittwald.de/keep-owner-references" StripLabels = "replicator.v1.mittwald.de/strip-labels" + StripAnnotations = "replicator.v1.mittwald.de/strip-annotations" ) diff --git a/replicate/service/services.go b/replicate/service/services.go new file mode 100644 index 00000000..82d59e75 --- /dev/null +++ b/replicate/service/services.go @@ -0,0 +1,235 @@ +package service + +import ( + "context" + "encoding/json" + "fmt" + "net" + "time" + + "github.com/mittwald/kubernetes-replicator/replicate/common" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" +) + +type Replicator struct { + *common.GenericReplicator +} + +// NewReplicator creates a new service replicator +func NewReplicator(client kubernetes.Interface, resyncPeriod time.Duration, allowAll bool) common.Replicator { + repl := Replicator{ + GenericReplicator: common.NewGenericReplicator(common.ReplicatorConfig{ + Kind: "Service", + ObjType: &corev1.Service{}, + AllowAll: allowAll, + ResyncPeriod: resyncPeriod, + Client: client, + ListFunc: func(lo metav1.ListOptions) (runtime.Object, error) { + return client.CoreV1().Services("").List(context.TODO(), lo) + }, + WatchFunc: func(lo metav1.ListOptions) (watch.Interface, error) { + return client.CoreV1().Services("").Watch(context.TODO(), lo) + }, + }), + } + repl.UpdateFuncs = common.UpdateFuncs{ + ReplicateObjectTo: repl.ReplicateObjectTo, + PatchDeleteDependent: repl.PatchDeleteDependent, + DeleteReplicatedResource: repl.DeleteReplicatedResource, + } + + return &repl +} + +// ReplicateObjectTo copies the whole object to target namespace +func (r *Replicator) ReplicateObjectTo(sourceObj interface{}, target *v1.Namespace) error { + source := sourceObj.(*corev1.Service) + targetLocation := fmt.Sprintf("%s/%s", target.Name, source.Name) + + logger := log. + WithField("kind", r.Kind). + WithField("source", common.MustGetKey(source)). + WithField("target", targetLocation) + + targetResource, exists, err := r.Store.GetByKey(targetLocation) + if err != nil { + return errors.Wrapf(err, "Could not get %s from cache!", targetLocation) + } + logger.Infof("Checking if %s exists? %v", targetLocation, exists) + + var targetCopy *corev1.Service + if exists { + targetObject := targetResource.(*corev1.Service) + targetVersion, ok := targetObject.Annotations[common.ReplicatedFromVersionAnnotation] + sourceVersion := source.ResourceVersion + + if ok && targetVersion == sourceVersion { + logger.Debugf("Service %s is already up-to-date", common.MustGetKey(targetObject)) + return nil + } + + targetCopy = targetObject.DeepCopy() + } else { + targetCopy = new(corev1.Service) + } + + keepOwnerReferences, ok := source.Annotations[common.KeepOwnerReferences] + if ok && keepOwnerReferences == "true" { + targetCopy.OwnerReferences = source.OwnerReferences + } + + if targetCopy.Annotations == nil { + targetCopy.Annotations = make(map[string]string) + } + + labelsCopy := make(map[string]string) + + stripLabels, ok := source.Annotations[common.StripLabels] + if !ok && stripLabels != "true" { + if source.Labels != nil { + for key, value := range source.Labels { + labelsCopy[key] = value + } + } + } + + annotationsCopy := make(map[string]string) + // we strip annotations by default as they usually contain data for eg. loadbalancer controllers + // a user has to set `"replicator.v1.mittwald.de/ strip-annotations = false"` to keep them + stripAnnotations, ok := source.Annotations[common.StripAnnotations] + if ok && stripAnnotations == "false" { + if source.Annotations != nil { + for key, value := range source.Annotations { + annotationsCopy[key] = value + } + } + } + + // we clean out .Spec and set our own + newSpec := new(corev1.ServiceSpec) + newSpec.Type = corev1.ServiceTypeExternalName + + // Get the full DNS name of the source service as cluster domains can vary + serviceFQDN, err := getFullDNSName(source.Name, source.Namespace) + if err != nil { + return errors.Wrapf(err, "Failed to get DNS name for service %s/%s", source.Namespace, source.Name) + } + logger.Debugf("Resolved existing service %s/%s to %s", target.Name, targetCopy.Name, serviceFQDN) + + newSpec.ExternalName = serviceFQDN + targetCopy.Name = source.Name + targetCopy.Labels = labelsCopy + targetCopy.Spec = *newSpec + targetCopy.Annotations = annotationsCopy + targetCopy.Annotations[common.ReplicatedAtAnnotation] = time.Now().Format(time.RFC3339) + targetCopy.Annotations[common.ReplicatedFromVersionAnnotation] = source.ResourceVersion + + var obj interface{} + + if exists { + if err == nil { + logger.Debugf("Updating existing service %s/%s", target.Name, targetCopy.Name) + obj, err = r.Client.CoreV1().Services(target.Name).Update(context.TODO(), targetCopy, metav1.UpdateOptions{}) + } + } else { + if err == nil { + logger.Debugf("Creating a new service %s/%s", target.Name, targetCopy.Name) + obj, err = r.Client.CoreV1().Services(target.Name).Create(context.TODO(), targetCopy, metav1.CreateOptions{}) + } + } + if err != nil { + return errors.Wrapf(err, "Failed to update service %s/%s", target.Name, targetCopy.Name) + } + + if err := r.Store.Update(obj); err != nil { + return errors.Wrapf(err, "Failed to update cache for %s/%s", target.Name, targetCopy) + } + + return nil +} + +func (r *Replicator) PatchDeleteDependent(sourceKey string, target interface{}) (interface{}, error) { + dependentKey := common.MustGetKey(target) + logger := log.WithFields(log.Fields{ + "kind": r.Kind, + "source": sourceKey, + "target": dependentKey, + }) + + targetObject, ok := target.(*corev1.Service) + if !ok { + err := errors.Errorf("bad type returned from Store: %T", target) + return nil, err + } + + patch := []common.JSONPatchOperation{{Operation: "remove", Path: "/imagePullSecrets"}} + patchBody, err := json.Marshal(&patch) + + if err != nil { + return nil, errors.Wrapf(err, "error while building patch body for service %s: %v", dependentKey, err) + + } + + logger.Debugf("clearing dependent service %s", dependentKey) + logger.Tracef("patch body: %s", string(patchBody)) + + s, err := r.Client.CoreV1().Services(targetObject.Namespace).Patch(context.TODO(), targetObject.Name, types.JSONPatchType, patchBody, metav1.PatchOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "error while patching service %s: %v", dependentKey, err) + } + return s, nil +} + +// DeleteReplicatedResource deletes a resource replicated by ReplicateTo annotation +func (r *Replicator) DeleteReplicatedResource(targetResource interface{}) error { + targetLocation := common.MustGetKey(targetResource) + logger := log.WithFields(log.Fields{ + "kind": r.Kind, + "target": targetLocation, + }) + + object := targetResource.(*corev1.Service) + logger.Debugf("Deleting %s", targetLocation) + if err := r.Client.CoreV1().Services(object.Namespace).Delete(context.TODO(), object.Name, metav1.DeleteOptions{}); err != nil { + return errors.Wrapf(err, "Failed deleting %s: %v", targetLocation, err) + } + return nil +} + +// Function to determine the full DNS name of the service +func getFullDNSName(serviceName, namespace string) (string, error) { + // Perform DNS lookup to get the IP address of the service + ips, err := net.LookupHost(fmt.Sprintf("%s.%s", serviceName, namespace)) + if err != nil { + // Return an empty string and the error if DNS lookup fails + return "", err + } + + // Check if the lookup returned at least one IP address + if len(ips) == 0 { + return "", fmt.Errorf("DNS lookup returned empty result") + } + + // Perform reverse DNS lookup to get the full DNS name of the IP address + names, err := net.LookupAddr(ips[0]) + if err != nil { + return "", err + } + + // Check if the reverse lookup returned at least one name + if len(names) == 0 { + return "", fmt.Errorf("reverse DNS lookup returned empty result") + } + + // Return the first name from the reverse lookup result + return names[0], nil +} diff --git a/test/configmap_source.yaml b/test/configmap_source.yaml index a299a273..4d591cfa 100644 --- a/test/configmap_source.yaml +++ b/test/configmap_source.yaml @@ -3,4 +3,13 @@ kind: ConfigMap metadata: name: config-source data: - foo: bar \ No newline at end of file + foo: bar +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-source-replicate-to + annotations: + replicator.v1.mittwald.de/replicate-to: some-namespace +data: + bar: foo \ No newline at end of file diff --git a/test/kustomization.yaml b/test/kustomization.yaml new file mode 100644 index 00000000..8beeabe4 --- /dev/null +++ b/test/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - configmap_reference.yaml + - configmap_source.yaml + - secret_reference.yaml + - secret_source.yaml + - service_source.yaml \ No newline at end of file diff --git a/test/secret_source.yaml b/test/secret_source.yaml index 0bb70199..ebfaa2ea 100644 --- a/test/secret_source.yaml +++ b/test/secret_source.yaml @@ -2,5 +2,9 @@ apiVersion: v1 kind: Secret metadata: name: source-secret + annotations: + # replicator.v1.mittwald.de/replication-allowed: "true" + # replicator.v1.mittwald.de/replication-allowed-namespaces: some-namespace + replicator.v1.mittwald.de/replicate-to: some-namespace data: value: SGFsbG8gV2VsdA== \ No newline at end of file diff --git a/test/service_source.yaml b/test/service_source.yaml new file mode 100644 index 00000000..4a121efc --- /dev/null +++ b/test/service_source.yaml @@ -0,0 +1,35 @@ +## this is the target but already exists +#apiVersion: v1 +#kind: Service +#metadata: +# name: source-service +# namespace: some-namespace +#spec: +# ports: +# - name: http +# port: 80 +# targetPort: http +# protocol: TCP +# selector: +# app: foo +# application: foo +#--- +apiVersion: v1 +kind: Service +metadata: + name: source-service + namespace: default + annotations: + alb.ingress.kubernetes.io/backend-protocol: HTTP + alb.ingress.kubernetes.io/healthcheck-path: /version + alb.ingress.kubernetes.io/healthcheck-protocol: HTTP + replicator.v1.mittwald.de/replicate-to: some-namespace +spec: + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP + selector: + app: foo + application: foo