diff --git a/Makefile b/Makefile index 440daace6..85b348007 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,10 @@ WATCH_NAMESPACE ?= "" IMG ?= $(IMAGE_REGISTRY)/$(REGISTRY_NAMESPACE)/$(IMAGE_NAME):$(IMAGE_TAG) +# NETWORK_POLICY controls whether NetworkPolicy manifests are included +# in the generated installer YAMLs. Set to true to enable. +NETWORK_POLICY ?= false + # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.29.0 @@ -74,6 +78,15 @@ resources: endef export BUILD_CSI_RBAC_OVERLAY +# Helpers to toggle the network-policy kustomize resource. +# Called from build targets when NETWORK_POLICY=true. +define enable-network-policy +$(if $(filter true,$(NETWORK_POLICY)),cd config/default && sed -i 's|#- ../network-policy|- ../network-policy|' kustomization.yaml) +endef +define disable-network-policy +$(if $(filter true,$(NETWORK_POLICY)),cd config/default && sed -i 's|^- ../network-policy|#- ../network-policy|' kustomization.yaml) +endef + .PHONY: all all: build @@ -192,19 +205,23 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform .PHONY: build-installer build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. + $(call enable-network-policy) mkdir -p build deploy/all-in-one cd build && echo "$$BUILD_INSTALLER_OVERLAY" > kustomization.yaml cd build && $(KUSTOMIZE) edit add resource ../config/default/ $(KUSTOMIZE) build build > deploy/all-in-one/install.yaml rm -rf build + $(call disable-network-policy) .PHONY: build-openshift-installer build-openshift-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs, deployment, and OpenShift SCC. + $(call enable-network-policy) mkdir -p build deploy/all-in-one cd build && echo "$$BUILD_INSTALLER_OVERLAY" > kustomization.yaml cd build && $(KUSTOMIZE) edit add resource ../config/openshift/ $(KUSTOMIZE) build build > deploy/all-in-one/install-openshift.yaml rm -rf build + $(call disable-network-policy) .PHONY: build-helm-installer build-helm-installer: manifests generate kustomize helmify ## Generate helm charts for the operator. @@ -275,11 +292,13 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + $(call enable-network-policy) mkdir -p build cd build && echo "$$BUILD_INSTALLER_OVERLAY" > kustomization.yaml cd build && $(KUSTOMIZE) edit add resource ../config/default/ $(KUSTOMIZE) build build | $(KUBECTL) apply -f - rm -rf build + $(call disable-network-policy) .PHONY: undeploy undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index c76f34bf7..c66e1f6c9 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -8,4 +8,5 @@ - Fencing can now be enabled in the ceph-csi-drivers Helm chart either globally for all drivers or on a per-driver basis. - Added `enabled` field to log rotation configuration in the ceph-csi-drivers Helm chart. Users can now disable log rotation by setting `log.rotation.enabled: false` either globally in `operatorConfig.driverSpecDefaults` or per-driver in `drivers.`. This resolves Helm warnings when disabling file logging from parent charts. - Added support to set the priorityClass in the operator helm chart. +- Added NetworkPolicies for the operator pod and CSI driver pods (controller-plugin, csi-addons node agent). Disabled by default; enabled with `NETWORK_POLICY=true` at build time. Driver NPs are created by the operator when `networking.k8s.io/v1` is available on the cluster. Node-plugin pods are exempt (`hostNetwork: true`). ## NOTE diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index b28b314c1..471e9cc6d 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -18,10 +18,8 @@ resources: #- ../prometheus # [METRICS] Expose the controller manager metrics service. # - metrics_service.yaml -# [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy. -# Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. -# Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will -# be able to communicate with the Webhook Server. +# [NETWORK POLICY] Protect the operator pod with NetworkPolicy. +# Denies all ingress and allows open egress for API server access. #- ../network-policy #patches: diff --git a/config/network-policy/allow-metrics-traffic.yaml b/config/network-policy/allow-metrics-traffic.yaml deleted file mode 100644 index 7491dba4a..000000000 --- a/config/network-policy/allow-metrics-traffic.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# This NetworkPolicy allows ingress traffic -# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those -# namespaces are able to gather data from the metrics endpoint. -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - labels: - app.kubernetes.io/name: kube - app.kubernetes.io/managed-by: kustomize - name: allow-metrics-traffic - namespace: system -spec: - podSelector: - matchLabels: - control-plane: controller-manager - app.kubernetes.io/name: kube - policyTypes: - - Ingress - ingress: - # This allows ingress traffic from any namespace with the label metrics: enabled - - from: - - namespaceSelector: - matchLabels: - metrics: enabled # Only from namespaces with this label - ports: - - port: 8443 - protocol: TCP diff --git a/config/network-policy/kustomization.yaml b/config/network-policy/kustomization.yaml index ec0fb5e57..fe98e9d73 100644 --- a/config/network-policy/kustomization.yaml +++ b/config/network-policy/kustomization.yaml @@ -1,2 +1,2 @@ resources: -- allow-metrics-traffic.yaml +- operator-network-policy.yaml diff --git a/config/network-policy/operator-network-policy.yaml b/config/network-policy/operator-network-policy.yaml new file mode 100644 index 000000000..575900450 --- /dev/null +++ b/config/network-policy/operator-network-policy.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: ceph-csi-controller-manager +spec: + podSelector: + matchLabels: + control-plane: ceph-csi-op-controller-manager + # Ingress is not specified — deny all inbound traffic. + egress: + - {} + policyTypes: + - Ingress + - Egress diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c21e9b59e..9371f5e9d 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -87,6 +87,18 @@ rules: - get - list - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - storage.k8s.io resources: diff --git a/deploy/all-in-one/install-openshift.yaml b/deploy/all-in-one/install-openshift.yaml index 1568b6e86..898a06047 100644 --- a/deploy/all-in-one/install-openshift.yaml +++ b/deploy/all-in-one/install-openshift.yaml @@ -15802,6 +15802,18 @@ rules: - get - list - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - storage.k8s.io resources: diff --git a/deploy/all-in-one/install.yaml b/deploy/all-in-one/install.yaml index 051297782..61d4f7768 100644 --- a/deploy/all-in-one/install.yaml +++ b/deploy/all-in-one/install.yaml @@ -15788,6 +15788,18 @@ rules: - get - list - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - storage.k8s.io resources: diff --git a/deploy/charts/ceph-csi-operator/templates/manager-rbac.yaml b/deploy/charts/ceph-csi-operator/templates/manager-rbac.yaml index b73f879ea..6b2cac177 100644 --- a/deploy/charts/ceph-csi-operator/templates/manager-rbac.yaml +++ b/deploy/charts/ceph-csi-operator/templates/manager-rbac.yaml @@ -88,6 +88,18 @@ rules: - get - list - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - storage.k8s.io resources: diff --git a/deploy/multifile/operator.yaml b/deploy/multifile/operator.yaml index b4ade75f4..28bf130eb 100644 --- a/deploy/multifile/operator.yaml +++ b/deploy/multifile/operator.yaml @@ -345,6 +345,18 @@ rules: - get - list - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - storage.k8s.io resources: diff --git a/docs/design/network-policy.md b/docs/design/network-policy.md new file mode 100644 index 000000000..2f2f32629 --- /dev/null +++ b/docs/design/network-policy.md @@ -0,0 +1,86 @@ +# NetworkPolicy for Ceph-CSI Operator and Driver Pods + +This document describes the NetworkPolicy implementation for the +ceph-csi-operator and the CSI driver pods it manages. + +## Component Inventory + +| Component | Pod Labels | hostNetwork | NP Coverage | +|---|---|---|---| +| Operator pod | `control-plane: ceph-csi-op-controller-manager` | No | Static manifest | +| Controller-plugin (per driver) | `app: {driver}-ctrlplugin` | Configurable | Code-generated | +| csi-addons node agent (per driver) | `app: {driver}-nodeplugin-csi-addons` | No | Code-generated | +| Node-plugin DaemonSet | `app: {driver}-nodeplugin` | Yes (hardcoded) | Exempt | + +## Policies + +### Operator Pod + +- **Ingress:** Deny all. No exposed ports (metrics disabled by default, + kubelet health probes bypass NetworkPolicy). +- **Egress:** Open (`- {}`). Required for API server access. The API server + ClusterIP varies per cluster (10.96.0.1 on kubeadm, 172.30.0.1 on OpenShift) + and CNI ipBlock behavior is inconsistent across implementations, so + restricting egress to a specific IP is not portable. + +### Controller-Plugin + +- **Ingress:** Two rules: + 1. csi-addons controller-manager → csi-addons gRPC port (9070 for RBD, + 9080 for CephFS). Source restricted to pods with labels + `app.kubernetes.io/name: csi-addons` + `control-plane: controller-manager` + in any namespace. + 2. (RBD only) Any pod → snapshot-metadata gRPC port 50051. Open source + because backup applications (Velero, etc.) can run in any namespace + with any labels. See [CSI snapshot-metadata](https://kubernetes-csi.github.io/docs/external-snapshot-metadata.html). +- **Egress:** Open. Required for API server and Ceph cluster access. + +### csi-addons Node Agent + +- **Ingress:** csi-addons controller-manager → port 9071 only. +- **Egress:** Open. +- **Lifecycle:** Mirrors the csi-addons DaemonSet — created when + `DeployCsiAddons` is true, deleted when false. Not created for NFS drivers. + +### Node-Plugin DaemonSet + +No NetworkPolicy. Runs with `hostNetwork: true`, which makes it exempt +from NetworkPolicy enforcement. + +## Gating + +### Static Manifest (Operator Pod) + +The operator pod NP is in `config/network-policy/` and referenced from +`config/default/kustomization.yaml` as a commented resource. It is +**disabled by default**. Set `NETWORK_POLICY=true` when building to include it: + +```console +NETWORK_POLICY=true make build-installer +``` + +### Code-Generated NPs (Driver Pods) + +Driver pod NPs are created by the operator at runtime. The operator checks +for the `networking.k8s.io/v1` API group at startup via the discovery client. +If the API is not available, no NPs are created and a log message is emitted. + +## API Server Egress Rationale + +Open egress (`- {}`) is used instead of restricting to a specific API server +IP because: + +1. The API server ClusterIP varies per cluster and is unknown at manifest + authoring time. +2. CNI implementations handle `ipBlock` inconsistently with DNAT (OVN-K8s + timing, Cilium quirks). +3. HyperShift clusters may use non-standard API server ports. + +This follows the precedent set by +[operator-marketplace PR #723](https://github.com/operator-framework/operator-marketplace/pull/723). + +## References + +- [CSI snapshot-metadata sidecar](https://kubernetes-csi.github.io/docs/external-snapshot-metadata.html) +- [kubernetes-csi/external-snapshot-metadata](https://github.com/kubernetes-csi/external-snapshot-metadata) +- [operator-marketplace NP precedent](https://github.com/operator-framework/operator-marketplace/pull/723) diff --git a/docs/kubernetes-installation.md b/docs/kubernetes-installation.md index 688cd7ccc..7ca3b3d44 100644 --- a/docs/kubernetes-installation.md +++ b/docs/kubernetes-installation.md @@ -495,3 +495,33 @@ Expected output: ```text No resources found in ceph-csi-operator-system namespace. ``` + +## 6. Optional: Enable NetworkPolicies + +NetworkPolicies for the operator pod and CSI driver pods are included but +**disabled by default**. To include them in the generated manifests, pass +`NETWORK_POLICY=true` to the build target: + +```bash +NETWORK_POLICY=true make build-installer +# or for OpenShift: +NETWORK_POLICY=true make build-openshift-installer +``` + +When enabled: + +- The **operator pod** gets a policy that denies all ingress and allows open + egress (required for API server access). +- Each **controller-plugin** pod gets a policy allowing ingress only from the + csi-addons controller-manager on the csi-addons gRPC port, and (for RBD) + from any pod on port 50051 for snapshot-metadata gRPC. +- Each **csi-addons node agent** pod (when `deployCsiAddons: true`) gets a + policy allowing ingress only from the csi-addons controller-manager on + port 9071. +- The **node-plugin** DaemonSet runs with `hostNetwork: true` and is exempt + from NetworkPolicies. + +Driver pod NetworkPolicies are created by the operator when the cluster +supports the `networking.k8s.io/v1` API group (checked at startup). + +For design details, see [docs/design/network-policy.md](design/network-policy.md). diff --git a/internal/controller/driver_controller.go b/internal/controller/driver_controller.go index 15286eb15..14ab4975f 100644 --- a/internal/controller/driver_controller.go +++ b/internal/controller/driver_controller.go @@ -32,11 +32,13 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" storagev1 "k8s.io/api/storage/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/discovery" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -62,6 +64,7 @@ import ( //+kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="",resources=nodes,verbs=list;watch //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch;delete type DriverType string @@ -93,7 +96,8 @@ var nameRegExp, _ = regexp.Compile(fmt.Sprintf( // DriverReconciler reconciles a Driver object type DriverReconciler struct { client.Client - Scheme *runtime.Scheme + Scheme *runtime.Scheme + networkPolicySupported bool } // A local reconcile object tied to a single reconcile iteration @@ -109,6 +113,24 @@ type driverReconcile struct { // SetupWithManager sets up the controller with the Manager. func (r *DriverReconciler) SetupWithManager(mgr ctrl.Manager) error { + setupLog := ctrl.Log.WithName("driver-controller-setup") + dc, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig()) + if err != nil { + setupLog.Info("Failed to create discovery client, NetworkPolicy support disabled", "error", err) + } else { + resources, err := dc.ServerResourcesForGroupVersion("networking.k8s.io/v1") + if err != nil { + setupLog.Info("NetworkPolicy API not found, driver NetworkPolicies will not be created", "error", err) + } else { + for _, res := range resources.APIResources { + if res.Kind == "NetworkPolicy" { + r.networkPolicySupported = true + break + } + } + } + } + // Define conditions for an OperatorConfig change that the require queuing of reconciliation // Filter update events based on metadata.generation changes, will filter events // for non-spec changes on most resource types. @@ -185,6 +207,10 @@ func (r *DriverReconciler) SetupWithManager(mgr ctrl.Manager) error { &corev1.Service{}, builder.WithPredicates(genChangedPredicate), ). + Owns( + &networkingv1.NetworkPolicy{}, + builder.WithPredicates(genChangedPredicate), + ). Owns( &corev1.ConfigMap{}, builder.MatchEveryOwner, @@ -241,9 +267,11 @@ func (r *driverReconcile) reconcile() error { r.reconcileLogRotateConfigMap, r.reconcileK8sCsiDriver, r.reconcileControllerPluginDeployment, + r.reconcileControllerPluginNetworkPolicy, r.reconcileNodePluginDaemonSet, r.reconcileLivenessService, r.reconcileNodePluginDaemonSetForCsiAddons, + r.reconcileNodeAgentNetworkPolicy, } // Concurrently reconcile different aspects of the clusters actual state to meet @@ -1041,6 +1069,65 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { return err } +func (r *driverReconcile) reconcileControllerPluginNetworkPolicy() error { + if !r.networkPolicySupported { + return nil + } + + np := &networkingv1.NetworkPolicy{} + np.Name = r.generateName("ctrlplugin") + np.Namespace = r.driver.Namespace + + log := r.log.WithValues("networkPolicy", np.Name) + log.Info("Reconciling controller plugin network policy") + + opResult, err := ctrlutil.CreateOrUpdate(r.ctx, r.Client, np, func() error { + if err := ctrlutil.SetControllerReference(&r.driver, np, r.Scheme); err != nil { + return err + } + + csiAddonsPort := r.controllerPluginCsiAddonsContainerPort() + proto := corev1.ProtocolTCP + + np.Spec = networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"app": np.Name}, + }, + Ingress: []networkingv1.NetworkPolicyIngressRule{ + { + From: []networkingv1.NetworkPolicyPeer{{ + NamespaceSelector: &metav1.LabelSelector{}, + PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": "csi-addons", + "control-plane": "controller-manager", + }, + }, + }}, + Ports: []networkingv1.NetworkPolicyPort{ + {Port: &intstr.IntOrString{Type: intstr.Int, IntVal: csiAddonsPort.ContainerPort}, Protocol: &proto}, + }, + }, + }, + Egress: []networkingv1.NetworkPolicyEgressRule{{}}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress}, + } + + if r.isRbdDriver() { + np.Spec.Ingress = append(np.Spec.Ingress, networkingv1.NetworkPolicyIngressRule{ + Ports: []networkingv1.NetworkPolicyPort{ + {Port: &intstr.IntOrString{Type: intstr.Int, IntVal: utils.SnapshotMetadataGrpcPort.ContainerPort}, Protocol: &proto}, + }, + }) + } + + return nil + }) + + logCreateOrUpdateResult(log, "NetworkPolicy", np, opResult, err) + return err +} + func (r *driverReconcile) controllerPluginCsiAddonsContainerPort() corev1.ContainerPort { // the cephFS and rbd drivers need to use different ports // to avoid port collisions with host network. @@ -1257,6 +1344,58 @@ func (r *driverReconcile) reconcileNodePluginDaemonSetForCsiAddons() error { return err } +func (r *driverReconcile) reconcileNodeAgentNetworkPolicy() error { + if !r.networkPolicySupported { + return nil + } + + np := &networkingv1.NetworkPolicy{} + np.Name = r.generateName("nodeplugin-csi-addons") + np.Namespace = r.driver.Namespace + + if !ptr.Deref(r.driver.Spec.DeployCsiAddons, false) || r.isNfsDriver() { + if err := r.Delete(r.ctx, np); client.IgnoreNotFound(err) != nil { + return err + } + return nil + } + + log := r.log.WithValues("networkPolicy", np.Name) + log.Info("Reconciling csi-addons node agent network policy") + + proto := corev1.ProtocolTCP + opResult, err := ctrlutil.CreateOrUpdate(r.ctx, r.Client, np, func() error { + if err := ctrlutil.SetControllerReference(&r.driver, np, r.Scheme); err != nil { + return err + } + np.Spec = networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"app": np.Name}, + }, + Ingress: []networkingv1.NetworkPolicyIngressRule{{ + From: []networkingv1.NetworkPolicyPeer{{ + NamespaceSelector: &metav1.LabelSelector{}, + PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": "csi-addons", + "control-plane": "controller-manager", + }, + }, + }}, + Ports: []networkingv1.NetworkPolicyPort{ + {Port: &intstr.IntOrString{Type: intstr.Int, IntVal: utils.NodePluginCsiAddonsContainerPort.ContainerPort}, Protocol: &proto}, + }, + }}, + Egress: []networkingv1.NetworkPolicyEgressRule{{}}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress}, + } + return nil + }) + + logCreateOrUpdateResult(log, "NetworkPolicy", np, opResult, err) + return err +} + func (r *driverReconcile) reconcileNodePluginDaemonSet() error { daemonSet := &appsv1.DaemonSet{} daemonSet.Name = r.generateName("nodeplugin")