Materialize CRD changes to use hash of the spec

Author: alex-hunt-materialize

Cargo.lock

@@ -290,6 +290,7 @@ aws-smithy-types = { version = "1.1.8", features = ["byte-stream-poll-next"] }
 aws-types = "1.3.9"
 axum = { version = "0.8.8", features = ["ws"] }
 axum-extra = { version = "0.12.5", features = ["typed-header"] }
+axum-server = { version = "0.8.0", features = ["tls-openssl"] }
 azure_core = "0.21.0"
 azure_identity = "0.21.0"
 azure_storage = "0.21.0"

Cargo.toml

@@ -2439,6 +2439,32 @@ steps:
         agents:
           queue: hetzner-aarch64-16cpu-32gb
 
+      - id: orchestratord-v1alpha2-opt-in
+        label: "Orchestratord v1alpha2 opt-in tests"
+        artifact_paths: ["mz_debug_*.zip"]
+        depends_on: devel-docker-tags
+        timeout_in_minutes: 120
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: orchestratord
+              run: v1alpha2-opt-in
+              ci-builder: stable
+        agents:
+          queue: hetzner-aarch64-16cpu-32gb
+
+      - id: orchestratord-manually-promote
+        label: "Orchestratord ManuallyPromote tests"
+        artifact_paths: ["mz_debug_*.zip"]
+        depends_on: devel-docker-tags
+        timeout_in_minutes: 120
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: orchestratord
+              run: manually-promote
+              ci-builder: stable
+        agents:
+          queue: hetzner-aarch64-16cpu-32gb
+
   - id: emulator
     label: Materialize Emulator
     depends_on: build-aarch64

ci/nightly/pipeline.template.yml

@@ -107,6 +107,19 @@ Starting in v26.0, Self-Managed Materialize requires a license key.
    kubectl get nodes --show-labels
    ```
 
+1. Install cert-manager
+
+   Cert-manager is used for generating TLS certificates needed by the materialize operator
+   for CRD conversion webhooks.
+
+   ```shell
+   helm install cert-manager oci://quay.io/jetstack/charts/cert-manager \
+       --version v1.19.2 \
+       --namespace cert-manager \
+       --create-namespace \
+       --set crds.enabled=true
+   ```
+
 1. To help you get started for local evaluation/testing, Materialize provides
    some sample configuration files. Download the sample configuration files from
    the Materialize repo:

doc/user/content/self-managed-deployments/installation/install-on-local-kind.md

@@ -144,6 +144,8 @@ The following table lists the configurable parameters of the Materialize operato
 | `operator.args.enableInternalStatementLogging` |  | ``true`` |
 | `operator.args.enableLicenseKeyChecks` |  | ``false`` |
 | `operator.args.startupLogFilter` | Log filtering settings for startup logs | ``"INFO,mz_orchestratord=TRACE"`` |
+| `operator.certificate.secretName` | Name of a secret in the operator's namespace containing ca.crt, tls.crt, and tls.key entries. Only used if `source` is "secret". | ``nil`` |
+| `operator.certificate.source` | Where to obtain the certificate for orchestratord. Valid values are 'cert-manager' and 'secret'. | ``"cert-manager"`` |
 | `operator.cloudProvider.providers.aws.accountID` | When using AWS, accountID is required | ``""`` |
 | `operator.cloudProvider.providers.aws.enabled` |  | ``false`` |
 | `operator.cloudProvider.providers.aws.iam.roles.connection` | ARN for CREATE CONNECTION feature | ``""`` |

misc/helm-charts/operator/README.md

@@ -0,0 +1,41 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+{{- if eq .Values.operator.certificate.source "cert-manager" -}}
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ include "materialize-operator.fullname" . }}-self-signed
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "materialize-operator.labels" . | nindent 4 }}
+spec:
+  selfSigned: {}
+
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "materialize-operator.fullname" . }}-self-signed
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "materialize-operator.labels" . | nindent 4 }}
+spec:
+  dnsNames:
+    - {{ include "materialize-operator.fullname" . }}.{{ .Release.Namespace }}.svc
+  secretName: {{ include "materialize-operator.fullname" . }}-cert
+  privateKey:
+    algorithm: ECDSA
+    rotationPolicy: Always
+  issuerRef:
+    name: {{ include "materialize-operator.fullname" . }}-self-signed
+    kind: Issuer
+    group: cert-manager.io
+{{- end -}}

misc/helm-charts/operator/templates/certificate.yaml

@@ -76,6 +76,7 @@ rules:
 - apiGroups: ["apiextensions.k8s.io"]
   resources:
   - customresourcedefinitions
+  - customresourcedefinitions/status
   verbs:
   - create
   - update

misc/helm-charts/operator/templates/clusterrole.yaml

@@ -242,9 +242,15 @@ spec:
         - >
           --additional-crd-columns={{ toJson .Values.operator.additionalMaterializeCRDColumns }}
         {{- end }}
+        - "--webhook-service-name"
+        - {{ include "materialize-operator.fullname" . }}
+        - "--webhook-service-namespace"
+        - {{ .Release.Namespace }}
         ports:
         - containerPort: 3100
           name: metrics
+        - containerPort: 8001
+          name: webhook
         resources:
           {{- toYaml .Values.operator.resources | nindent 10 }}
         securityContext:
@@ -256,3 +262,27 @@ spec:
           runAsNonRoot: true
           seccompProfile:
             type: RuntimeDefault
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: webhook
+            scheme: HTTPS
+          failureThreshold: 3
+          periodSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: webhook
+            scheme: HTTPS
+          failureThreshold: 1
+          periodSeconds: 10
+        volumeMounts:
+          - mountPath: /etc/tls
+            name: certificate
+            readOnly: true
+      volumes:
+        - name: certificate
+          secret:
+            defaultMode: 256
+            optional: false
+            secretName: {{ if eq .Values.operator.certificate.source "cert-manager" }}{{ include "materialize-operator.fullname" . }}-cert{{ else }}{{ .Values.operator.certificate.secretName }}{{ end }}

misc/helm-charts/operator/templates/deployment.yaml

@@ -0,0 +1,25 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "materialize-operator.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "materialize-operator.labels" . | nindent 4 }}
+spec:
+  selector:
+    {{- include "materialize-operator.selectorLabels" . | nindent 4 }}
+  ports:
+    - name: webhook
+      protocol: TCP
+      port: 8001
+      targetPort: 8001

misc/helm-charts/operator/templates/service.yaml

@@ -34,6 +34,13 @@ operator:
     #  priority: 2
     #  type: "string"
 
+  certificate:
+    # -- (string) Where to obtain the certificate for orchestratord. Valid values are 'cert-manager' and 'secret'.
+    source: cert-manager
+    # -- (string) Name of a secret in the operator's namespace containing ca.crt, tls.crt, and tls.key entries. Only used if `source` is "secret".
+    secretName: null
+
+
   # Cloud provider configuration
   cloudProvider:
     # -- Specifies cloud provider. Valid values are 'aws', 'gcp', 'azure' , 'generic', or 'local'