Materialize CRD changes to use hash of the spec

Author: alex-hunt-materialize

Cargo.lock

@@ -2541,3 +2541,29 @@ steps:
               ci-builder: stable
         agents:
           queue: hetzner-aarch64-16cpu-32gb
+
+      - id: orchestratord-mid-rollout-migration
+        label: "Orchestratord mid-rollout CRD migration tests"
+        artifact_paths: ["mz_debug_*.zip"]
+        depends_on: devel-docker-tags
+        timeout_in_minutes: 120
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: orchestratord
+              run: mid-rollout-migration
+              ci-builder: stable
+        agents:
+          queue: hetzner-aarch64-16cpu-32gb
+
+      - id: orchestratord-force-rollout-annotation
+        label: "Orchestratord force-rollout annotation tests"
+        artifact_paths: ["mz_debug_*.zip"]
+        depends_on: devel-docker-tags
+        timeout_in_minutes: 120
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: orchestratord
+              run: force-rollout-annotation
+              ci-builder: stable
+        agents:
+          queue: hetzner-aarch64-16cpu-32gb

ci/nightly/pipeline.template.yml

@@ -107,6 +107,19 @@ Starting in v26.0, Self-Managed Materialize requires a license key.
    kubectl get nodes --show-labels
    ```
 
+1. Install cert-manager
+
+   Cert-manager is used for generating TLS certificates needed by the materialize operator
+   for CRD conversion webhooks.
+
+   ```shell
+   helm install cert-manager oci://quay.io/jetstack/charts/cert-manager \
+       --version v1.19.2 \
+       --namespace cert-manager \
+       --create-namespace \
+       --set crds.enabled=true
+   ```
+
 1. To help you get started for local evaluation/testing, Materialize provides
    some sample configuration files. Download the sample configuration files from
    the Materialize repo:

doc/user/content/self-managed-deployments/installation/install-on-local-kind.md

@@ -144,6 +144,8 @@ The following table lists the configurable parameters of the Materialize operato
 | `operator.args.enableInternalStatementLogging` |  | ``true`` |
 | `operator.args.enableLicenseKeyChecks` |  | ``false`` |
 | `operator.args.startupLogFilter` | Log filtering settings for startup logs | ``"INFO,mz_orchestratord=TRACE"`` |
+| `operator.certificate.secretName` | Name of a secret in the operator's namespace containing ca.crt, tls.crt, and tls.key entries. Only used if `source` is "secret". | ``nil`` |
+| `operator.certificate.source` | Where to obtain the certificate for orchestratord. Valid values are 'cert-manager' and 'secret'. | ``"cert-manager"`` |
 | `operator.cloudProvider.providers.aws.accountID` | When using AWS, accountID is required | ``""`` |
 | `operator.cloudProvider.providers.aws.enabled` |  | ``false`` |
 | `operator.cloudProvider.providers.aws.iam.roles.connection` | ARN for CREATE CONNECTION feature | ``""`` |

misc/helm-charts/operator/README.md

@@ -0,0 +1,41 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+{{- if eq .Values.operator.certificate.source "cert-manager" -}}
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ include "materialize-operator.fullname" . }}-self-signed
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "materialize-operator.labels" . | nindent 4 }}
+spec:
+  selfSigned: {}
+
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "materialize-operator.fullname" . }}-self-signed
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "materialize-operator.labels" . | nindent 4 }}
+spec:
+  dnsNames:
+    - {{ include "materialize-operator.fullname" . }}.{{ .Release.Namespace }}.svc
+  secretName: {{ include "materialize-operator.fullname" . }}-cert
+  privateKey:
+    algorithm: Ed25519
+    rotationPolicy: Always
+  issuerRef:
+    name: {{ include "materialize-operator.fullname" . }}-self-signed
+    kind: Issuer
+    group: cert-manager.io
+{{- end -}}

misc/helm-charts/operator/templates/certificate.yaml

@@ -240,9 +240,15 @@ spec:
         - >
           --additional-crd-columns={{ toJson .Values.operator.additionalMaterializeCRDColumns }}
         {{- end }}
+        - "--webhook-service-name"
+        - {{ include "materialize-operator.fullname" . }}
+        - "--webhook-service-namespace"
+        - {{ .Release.Namespace }}
         ports:
         - containerPort: 3100
           name: metrics
+        - containerPort: 8001
+          name: webhook
         resources:
           {{- toYaml .Values.operator.resources | nindent 10 }}
         securityContext:
@@ -254,3 +260,27 @@ spec:
           runAsNonRoot: true
           seccompProfile:
             type: RuntimeDefault
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: webhook
+            scheme: HTTPS
+          failureThreshold: 3
+          periodSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: webhook
+            scheme: HTTPS
+          failureThreshold: 1
+          periodSeconds: 10
+        volumeMounts:
+          - mountPath: /etc/tls
+            name: certificate
+            readOnly: true
+      volumes:
+        - name: certificate
+          secret:
+            defaultMode: 256
+            optional: false
+            secretName: {{ if eq .Values.operator.certificate.source "cert-manager" }}{{ include "materialize-operator.fullname" . }}-cert{{ else }}{{ .Values.operator.certificate.secretName }}{{ end }}

misc/helm-charts/operator/templates/deployment.yaml

@@ -0,0 +1,26 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "materialize-operator.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "materialize-operator.labels" . | nindent 4 }}
+spec:
+  selector:
+    {{- include "materialize-operator.selectorLabels" . | nindent 4 }}
+  ports:
+    - name: webhook
+      protocol: TCP
+      # TODO use a variable here
+      port: 8001
+      targetPort: 8001

misc/helm-charts/operator/templates/service.yaml

@@ -34,6 +34,13 @@ operator:
     #  priority: 2
     #  type: "string"
 
+  certificate:
+    # -- (string) Where to obtain the certificate for orchestratord. Valid values are 'cert-manager' and 'secret'.
+    source: cert-manager
+    # -- (string) Name of a secret in the operator's namespace containing ca.crt, tls.crt, and tls.key entries. Only used if `source` is "secret".
+    secretName: null
+
+
   # Cloud provider configuration
   cloudProvider:
     # -- Specifies cloud provider. Valid values are 'aws', 'gcp', 'azure' , 'generic', or 'local'

misc/helm-charts/operator/values.yaml

@@ -27,6 +27,7 @@ schemars = { version = "1.2.1", features = ["uuid1"] }
 semver = "1.0.27"
 serde = "1.0.219"
 serde_json = "1.0.149"
+sha2 = "0.10.9"
 tracing = "0.1.44"
 uuid = { version = "1.19", features = ["serde", "v4"] }
 workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true }