From d9714d023644ab7f57c8ccd0439224966f14cf1e Mon Sep 17 00:00:00 2001 From: Alex Hunt Date: Tue, 3 Mar 2026 11:32:19 +0000 Subject: [PATCH 1/6] update docs for Materialize CRD v1 --- bin/bump-version | 8 +- doc/user/content/releases/_index.md | 4 +- .../security/self-managed/authentication.md | 74 +++ doc/user/content/security/self-managed/sso.md | 2 +- .../self-managed-deployments/_index.md | 32 + .../configuration-system-parameters.md | 42 ++ .../_index.md} | 6 +- .../materialize-crd-field-descriptions/v1.md | 10 + .../v1alpha1.md | 10 + .../upgrading/_index.md | 220 +------ .../upgrading/materialize-instances/_index.md | 86 +++ .../upgrading/materialize-instances/v1.md | 173 ++++++ .../materialize-instances/v1alpha1.md | 234 ++++++++ .../materialize_crd_descriptions_v1.json | 558 ++++++++++++++++++ ...aterialize_crd_descriptions_v1alpha1.json} | 0 doc/user/data/self_managed/upgrades.yml | 44 +- ...l => materialize-crd-descriptions-v1.html} | 4 +- ...materialize-crd-descriptions-v1alpha1.html | 57 ++ .../upgrade/upgrade-steps-local-kind.html | 31 + .../self-managed/upgrade-notes/v26.0.md | 2 +- .../self-managed/upgrade-notes/v26.30.md | 287 +++++++++ src/cloud-resources/src/bin/crd_writer.rs | 13 +- src/cloud-resources/src/crd/materialize.rs | 2 +- 23 files changed, 1668 insertions(+), 231 deletions(-) rename doc/user/content/self-managed-deployments/{materialize-crd-field-descriptions.md => materialize-crd-field-descriptions/_index.md} (52%) create mode 100644 doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md create mode 100644 doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md create mode 100644 doc/user/content/self-managed-deployments/upgrading/materialize-instances/_index.md create mode 100644 doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1.md create mode 100644 doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1alpha1.md create mode 100644 doc/user/data/self_managed/materialize_crd_descriptions_v1.json rename doc/user/data/self_managed/{materialize_crd_descriptions.json => materialize_crd_descriptions_v1alpha1.json} (100%) rename doc/user/layouts/shortcodes/self-managed/{materialize-crd-descriptions.html => materialize-crd-descriptions-v1.html} (89%) create mode 100644 doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions-v1alpha1.html create mode 100644 doc/user/shared-content/self-managed/upgrade-notes/v26.30.md diff --git a/bin/bump-version b/bin/bump-version index 1b6379f4c428e..96050379d276f 100755 --- a/bin/bump-version +++ b/bin/bump-version @@ -70,9 +70,11 @@ rm -f src/{clusterd,environmentd,materialized,persist-client,testdrive,catalog-d cargo update --workspace -crd_descriptions_json=doc/user/data/self_managed/materialize_crd_descriptions.json -cargo run -p mz-cloud-resources --bin crd-writer > "${crd_descriptions_json}" -git add "${crd_descriptions_json}" +for crd_version in v1alpha1 v1; do + crd_descriptions_json="doc/user/data/self_managed/materialize_crd_descriptions_${crd_version}.json" + cargo run -p mz-cloud-resources --bin crd-writer -- "${crd_version}" > "${crd_descriptions_json}" + git add "${crd_descriptions_json}" +done bin/helm-chart-version-bump --bump-orchestratord-version "v$version" diff --git a/doc/user/content/releases/_index.md b/doc/user/content/releases/_index.md index bb9953890e8d9..dae05f381aca0 100644 --- a/doc/user/content/releases/_index.md +++ b/doc/user/content/releases/_index.md @@ -1204,7 +1204,7 @@ v26.4.0 introduces several performance improvements and bugfixes. - **Up to 3x faster hydration times for large PostgreSQL tables**: We've reduced the overhead incurred by communication between multiple *workers* on a large cluster. We've observed up to 3x throughput improvement when ingesting 1 TB PostgreSQL tables on large clusters. - **More efficient source ingestion batching**: Sources now batch writes more effectively. This can result in improved freshness and lower resource utilization, especially when a source is doing a large number of writes. - **CloudSQL HA failover support** (*Materialize Self-Managed only*): Materialize Self-Managed now offers better support for handling failovers in CloudSQL HA sources, without downtime. [Contact our support team](/support/) to enable this in your environment. -- **Manual Promotion** (*Materialize Self-Managed only*): [Rollout strategies](/self-managed-deployments/upgrading/#rollout-strategies) allow you control how Materialize transitions from the current generation to a new generation during an upgrade. We've added a new rollout strategy called `ManuallyPromote` which allows you to choose when to promote the new generation. This means that you can minimize the impact of potential downtime. +- **Manual Promotion** (*Materialize Self-Managed only*): [Rollout strategies](/self-managed-deployments/upgrading/materialize-instances/v1/#rollout-strategies) allow you control how Materialize transitions from the current generation to a new generation during an upgrade. We've added a new rollout strategy called `ManuallyPromote` which allows you to choose when to promote the new generation. This means that you can minimize the impact of potential downtime. ### Bug Fixes {#v26.4-bug-fixes} - Fixed timestamp determination logic to handle empty read holds correctly. @@ -1373,7 +1373,7 @@ use the new setting `rolloutStrategy` to specify either: - `WaitUntilReady` (*Default*) - `ImmediatelyPromoteCausingDowntime` -For more information, see [`rolloutStrategy`](/self-managed-deployments/upgrading/#rollout-strategies). +For more information, see [`rolloutStrategy`](/self-managed-deployments/upgrading/materialize-instances/v1/#rollout-strategies). ### Terraform helpers diff --git a/doc/user/content/security/self-managed/authentication.md b/doc/user/content/security/self-managed/authentication.md index 6e48e7af5c7b0..0d1361c109995 100644 --- a/doc/user/content/security/self-managed/authentication.md +++ b/doc/user/content/security/self-managed/authentication.md @@ -50,6 +50,40 @@ following fields: The following example Kubernetes manifest includes configuration for SASL/SCRAM-SHA-256 authentication: +{{< tabs >}} +{{< tab "v1 (v26.30+)" >}} + +```hc {hl_lines="15 25"} +apiVersion: v1 +kind: Namespace +metadata: + name: materialize-environment +--- +apiVersion: v1 +kind: Secret +metadata: + name: materialize-backend + namespace: materialize-environment +stringData: + metadata_backend_url: "..." + persist_backend_url: "..." + license_key: "..." + external_login_password_mz_system: "enter_mz_system_password" +--- +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:v26.12.1 + backendSecretName: materialize-backend + authenticatorKind: Sasl +``` + +{{< /tab >}} +{{< tab "v1alpha1 (before v26.30)" >}} + ```hc {hl_lines="15 25"} apiVersion: v1 kind: Namespace @@ -78,6 +112,9 @@ spec: authenticatorKind: Sasl ``` +{{< /tab >}} +{{< /tabs >}} + {{% include-headless "/headless/self-managed-deployments/enabled-auth-setting-warning" %}} @@ -97,6 +134,40 @@ To configure Self-Managed Materialize for password authentication, update the fo The following example Kubernetes manifest includes configuration for password authentication: +{{< tabs >}} +{{< tab "v1 (v26.30+)" >}} + +```hc {hl_lines="15 25"} +apiVersion: v1 +kind: Namespace +metadata: + name: materialize-environment +--- +apiVersion: v1 +kind: Secret +metadata: + name: materialize-backend + namespace: materialize-environment +stringData: + metadata_backend_url: "..." + persist_backend_url: "..." + license_key: "..." + external_login_password_mz_system: "enter_mz_system_password" +--- +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:v26.12.1 + backendSecretName: materialize-backend + authenticatorKind: Password +``` + +{{< /tab >}} +{{< tab "v1alpha1 (before v26.30)" >}} + ```hc {hl_lines="15 25"} apiVersion: v1 kind: Namespace @@ -125,6 +196,9 @@ spec: authenticatorKind: Password ``` +{{< /tab >}} +{{< /tabs >}} + {{% include-headless "/headless/self-managed-deployments/enabled-auth-setting-warning" %}} diff --git a/doc/user/content/security/self-managed/sso.md b/doc/user/content/security/self-managed/sso.md index 451c7b9ebff17..d1bb8cae88fed 100644 --- a/doc/user/content/security/self-managed/sso.md +++ b/doc/user/content/security/self-managed/sso.md @@ -254,7 +254,7 @@ spec: ``` Apply the updated manifest to your Kubernetes cluster. See -[Upgrading](/self-managed-deployments/upgrading/#rollout-configuration) for +[Upgrading](/self-managed-deployments/upgrading/materialize-instances/v1alpha1/#rollout-configuration) for details on rollout configuration. {{% include-headless diff --git a/doc/user/content/self-managed-deployments/_index.md b/doc/user/content/self-managed-deployments/_index.md index 8965aea956ab4..0980528be7ee4 100644 --- a/doc/user/content/self-managed-deployments/_index.md +++ b/doc/user/content/self-managed-deployments/_index.md @@ -157,6 +157,23 @@ custom resource definitions(CRDs). For a full list of fields available for the Materialize CR, see [Materialize CRD Field Descriptions](/self-managed-deployments/materialize-crd-field-descriptions/). +{{< tabs >}} +{{< tab "v1 (v26.30+)" >}} + +```yaml +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} +# ... additional fields omitted for brevity +``` + +{{< /tab >}} +{{< tab "v1alpha1 (before v26.30)" >}} + ```yaml apiVersion: materialize.cloud/v1alpha1 kind: Materialize @@ -168,11 +185,23 @@ spec: # ... additional fields omitted for brevity ``` +{{< /tab >}} +{{< /tabs >}} + When you first apply the Materialize custom resource, the operator automatically creates all required Kubernetes resources. #### Modifying the custom resource +{{< tabs >}} +{{< tab "v1 (v26.30+)" >}} + +To modify a custom resource, update the CRD with your changes. +When you apply the CRD, the operator will roll out the changes. + +{{< /tab >}} +{{< tab "v1alpha1 (before v26.30)" >}} + To modify a custom resource, update the CRD with your changes, including the `requestRollout` field with a new UUID value. When you apply the CRD, the operator will roll out the changes. @@ -182,6 +211,9 @@ If you do not specify a new `requestRollout` UUID, the operator watches for updates but does not roll out the changes. {{< /note >}} +{{< /tab >}} +{{< /tabs >}} + For a full list of fields available for the Materialize CR, see [Materialize CRD Field Descriptions](/self-managed-deployments/materialize-crd-field-descriptions/). diff --git a/doc/user/content/self-managed-deployments/configuration-system-parameters.md b/doc/user/content/self-managed-deployments/configuration-system-parameters.md index 5a1604f228f4d..44aa90b7bbe23 100644 --- a/doc/user/content/self-managed-deployments/configuration-system-parameters.md +++ b/doc/user/content/self-managed-deployments/configuration-system-parameters.md @@ -67,6 +67,24 @@ kubectl apply -f system-params-configmap.yaml Reference the ConfigMap in your Materialize custom resource by setting the `systemParameterConfigmapName` field to the name of your ConfigMap: +{{< tabs >}} +{{< tab "v1 (v26.30+)" >}} + +```yaml {hl_lines="9"} +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:v26.0.0 + backendSecretName: materialize-backend + systemParameterConfigmapName: mz-system-params +``` + +{{< /tab >}} +{{< tab "v1alpha1 (before v26.30)" >}} + ```yaml {hl_lines="9-10"} apiVersion: materialize.cloud/v1alpha1 kind: Materialize @@ -80,6 +98,9 @@ spec: requestRollout: 00000000-0000-0000-0000-000000000003 # Changing the CR requires a rollout ``` +{{< /tab >}} +{{< /tabs >}} + Apply the updated Materialize resource: ```shell @@ -129,6 +150,24 @@ Alternatively, you can add the `configmap-reload-trigger` annotation to your Materialize custom resource YAML and update it whenever you need to force a ConfigMap reload: +{{< tabs >}} +{{< tab "v1 (v26.30+)" >}} + +```yaml +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment + annotations: + configmap-reload-trigger: "1234567890" # Update this value to force reload +spec: + # ... rest of spec +``` + +{{< /tab >}} +{{< tab "v1alpha1 (before v26.30)" >}} + ```yaml apiVersion: materialize.cloud/v1alpha1 kind: Materialize @@ -141,6 +180,9 @@ spec: # ... rest of spec ``` +{{< /tab >}} +{{< /tabs >}} + {{< note >}} Even after the ConfigMap is synced, some system parameters may require a restart to take effect. diff --git a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions.md b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/_index.md similarity index 52% rename from doc/user/content/self-managed-deployments/materialize-crd-field-descriptions.md rename to doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/_index.md index bca28ab7a50d5..35ada81f0c88d 100644 --- a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions.md +++ b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/_index.md @@ -1,6 +1,7 @@ --- title: "Materialize CRD Field Descriptions" description: "Reference page on Materialize CRD Fields" +disable_list: true menu: main: parent: "sm-deployments" @@ -10,4 +11,7 @@ aliases: - /installation/appendix-materialize-crd-field-descriptions/ --- -{{% self-managed/materialize-crd-descriptions %}} +Select the CRD API version for your Materialize deployment: + +- [v1 (v26.30+)](/self-managed-deployments/materialize-crd-field-descriptions/v1/) +- [v1alpha1 (before v26.30)](/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1/) diff --git a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md new file mode 100644 index 0000000000000..403155796ec5c --- /dev/null +++ b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md @@ -0,0 +1,10 @@ +--- +title: "v1" +description: "Reference page on Materialize CRD Fields for the v1 API (v26.30+)" +menu: + main: + parent: "materialize-crd-field-descriptions" + weight: 10 +--- + +{{% self-managed/materialize-crd-descriptions-v1 %}} diff --git a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md new file mode 100644 index 0000000000000..41959aaa02545 --- /dev/null +++ b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md @@ -0,0 +1,10 @@ +--- +title: "v1alpha1" +description: "Reference page on Materialize CRD Fields for the v1alpha1 API (before v26.30)" +menu: + main: + parent: "materialize-crd-field-descriptions" + weight: 20 +--- + +{{% self-managed/materialize-crd-descriptions-v1alpha1 %}} diff --git a/doc/user/content/self-managed-deployments/upgrading/_index.md b/doc/user/content/self-managed-deployments/upgrading/_index.md index 9471a91bf58a8..77b5326918b40 100644 --- a/doc/user/content/self-managed-deployments/upgrading/_index.md +++ b/doc/user/content/self-managed-deployments/upgrading/_index.md @@ -93,224 +93,16 @@ helm upgrade -n materialize my-demo materialize/operator \ ## Upgrading Materialize Instances -**After** you have upgraded your Materialize Operator, upgrade your Materialize -instance(s) to the **APP Version** of the Operator. To find the version of your -currently deployed Materialize Operator: +Select the instructions for your CRD API version: -```shell -helm list -n materialize -``` - -You will use the returned **App Version** for the updated `environmentdImageRef` -value. Specifically, for your Materialize instance(s), set -`environmentdImageRef` value to use the new version: - -``` -spec: - environmentdImageRef: docker.io/materialize/environmentd: -``` - -To minimize unexpected downtime and avoid connection drops at critical -periods for your application, the upgrade process involves two steps: - -- First, stage the changes (update the `environmentdImageRef` with the new - version) to the Materialize custom resource. The Operator watches for changes - but does not automatically roll out the changes. - -- Second, roll out the changes by specifying a new UUID for `requestRollout`. - -### Stage the Materialize instance version change - -To stage the Materialize instances version upgrade, update the -`environmentdImageRef` field in the Materialize custom resource spec to the -compatible version of your currently deployed Materialize Operator. - -To stage, but **not** rollout, the Materialize instance version upgrade, you can -use the `kubectl patch` command; for example, if the **App Version** is {{< self-managed/versions/get-latest-version >}}: - -```shell -kubectl patch materialize \ - -n \ - --type='merge' \ - -p "{\"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\"}}" -``` - -{{< note >}} -Until you specify a new `requestRollout`, the Operator watches for updates but -does not roll out the changes. -{{< /note >}} - - -### Applying the changes via `requestRollout` - -To apply chang Materialize instance upgrade, you must update the `requestRollout` field in the Materialize custom resource spec to a new UUID. -Be sure to consult the [Rollout Configurations](#rollout-configuration) to ensure you've selected the correct rollout behavior. -```shell -# Then trigger the rollout with a new UUID -kubectl patch materialize \ - -n \ - --type='merge' \ - -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\"}}" -``` - -### Staging and applying in a single command - -Although separating the staging and rollout of the changes into two steps can -minimize unexpected downtime and avoid connection drops at critical periods, you -can, if preferred, combine both operations in a single command - -```shell -kubectl patch materialize \ - -n materialize-environment \ - --type='merge' \ - -p "{\"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\", \"requestRollout\": \"$(uuidgen)\"}}" -``` - -#### Using YAML Definition - -Alternatively, you can update your Materialize custom resource definition directly: - -```yaml -apiVersion: materialize.cloud/v1alpha1 -kind: Materialize -metadata: - name: 12345678-1234-1234-1234-123456789012 - namespace: materialize-environment -spec: - environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} # Update version as needed - requestRollout: 22222222-2222-2222-2222-222222222222 # Use a new UUID - forceRollout: 33333333-3333-3333-3333-333333333333 # Optional: for forced rollouts - inPlaceRollout: false # In Place rollout is deprecated and ignored. Please use rolloutStrategy - rolloutStrategy: WaitUntilReady # The mechanism to use when rolling out the new version. Can be WaitUntilReady or ImmediatelyPromoteCausingDowntime - backendSecretName: materialize-backend -``` - -Apply the updated definition: - -```shell -kubectl apply -f materialize.yaml -``` - -## Rollout Configuration +- [v1 (v26.30+)](/self-managed-deployments/upgrading/materialize-instances/v1/) +- [v1alpha1 (before v26.30)](/self-managed-deployments/upgrading/materialize-instances/v1alpha1/) -### `requestRollout` - -Specify a new `UUID` value for the `requestRollout` to roll out the changes to -the Materialize instance. - -{{< note >}} - -`requestRollout` without the `forcedRollout` field only rolls out if changes -exist to the Materialize instance. To roll out even if there are no changes to -the instance, use with `forcedRollouts`. - -{{< /note >}} - -```shell -# Only rolls out if there are changes -kubectl patch materialize \ - -n \ - --type='merge' \ - -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\"}}" -``` -#### `requestRollout` with `forcedRollouts` - -Specify a new `UUID` value for `forcedRollout` to roll out even when there are -no changes to the instance. Use `forcedRollout` with `requestRollout`. - -```shell -kubectl patch materialize \ - -n materialize-environment \ - --type='merge' \ - -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\", \"forceRollout\": \"$(uuidgen)\"}}" -``` - -### Rollout strategies - -Rollout strategies control how Materialize transitions from the current generation to a new generation during an upgrade. - -The behavior of the new version rollout follows your `rolloutStrategy` setting. - -#### *WaitUntilReady* - ***Default*** - -`WaitUntilReady` creates a new generation of pods and automatically promotes -them as soon as they catch up to the old generation and become `ReadyToPromote`. -Because both generations run simultaneously until the promotion, this strategy -temporarily doubles the required resources to run Materialize. - -{{< note >}} - -Starting in v26.28.0, `WaitUntilReady` waits up to 1 year (configurable by the -`with_0dt_deployment_max_wait` flag) for the new pods to become -`ReadyToPromote`. If the pods do not reach `ReadyToPromote` by then, the rollout -is cancelled and the new pods are removed. - -Earlier versions used a default wait of 72 hours, and, once the wait time -expired, the new pods were automatically promoted regardless of their readiness. -{{< /note >}} - -#### *ImmediatelyPromoteCausingDowntime* -{{< warning >}} Using the `ImmediatelyPromoteCausingDowntime` rollout flag will cause downtime. {{< /warning >}} - -`ImmediatelyPromoteCausingDowntime` tears down the prior generation, and immediately promotes the new generation without waiting for it to hydrate. This causes downtime until the new generation has hydrated. However, it does not require additional resources. - -#### *ManuallyPromote* - -`ManuallyPromote` allows you to choose when to promote the new generation. This means you can time the promotion for periods when load is low, minimizing the impact of potential downtime for any clients connected to Materialize. This strategy temporarily doubles the required resources to run Materialize. - -To minimize downtime, wait until the new generation has fully hydrated and caught up to the prior generation before promoting. To check hydration status, inspect the `UpToDate` condition in the Materialize resource status. When hydration completes, the condition will be `ReadyToPromote`. - -To promote, update the `forcePromote` field to match the `requestRollout` field in the Materialize spec. If you need to promote before hydration completes, you can set `forcePromote` immediately, but clients may experience downtime. - -{{< warning >}} Leaving a new generation unpromoted for over 6 hours may cause downtime. {{< /warning >}} - -**Do not leave new generations unpromoted indefinitely**. They should either be promoted or canceled. New generations open a read hold on the metadata database that prevents compaction. This hold is only released when the generation is promoted or canceled. If left open too long, promoting or canceling can trigger a spike in deletion load on the metadata database, potentially causing downtime. It is not recommended to leave generations unpromoted for over 6 hours. - -#### *inPlaceRollout* - ***Deprecated*** - -The setting is ignored. - -## Verifying the Upgrade - -After initiating the rollout, you can monitor the status field of the Materialize custom resource to check on the upgrade. - -```shell -# Watch the status of your Materialize environment -kubectl get materialize -n materialize-environment -w - -# Check the logs of the operator -kubectl logs -l app.kubernetes.io/name=materialize-operator -n materialize -``` - -## Cancelling the Upgrade - -You may want to cancel an in-progress rollout if the upgrade has failed. This may be indicated by new pods not being healthy. Before cancelling, verify that the upgrade has not already completed by checking that the deploy generation (found via `status.activeGeneration`) is still the one from before the upgrade. Once an upgrade has already happened, you cannot revert using this method. - -To cancel an in-progress rollout and revert to the last completed rollout -state, revert both `requestRollout` and `environmentdImageRef` back to the -values from the last completed rollout. Reverting `environmentdImageRef` -alongside `requestRollout` keeps the spec aligned with what is actually -running, so a later rollout doesn't accidentally pick up the previously -attempted upgrade image. - -First, retrieve the last completed rollout request ID and the matching -environmentd image ref from your Materialize CR: - -```shell -kubectl get materialize -n materialize-environment \ - -o jsonpath='{.status.lastCompletedRolloutRequest} {.status.lastCompletedRolloutEnvironmentdImageRef}' -``` +## Version Specific Upgrade Notes -Then, set both fields back to these values in a single patch: +### Upgrading to `v26.30` and later versions -```shell -kubectl patch materialize \ - -n materialize-environment \ - --type='merge' \ - -p "{\"spec\": {\"requestRollout\": \"\", \"environmentdImageRef\": \"\"}}" -``` - -## Version Specific Upgrade Notes +{{< include-md file="shared-content/self-managed/upgrade-notes/v26.30.md" >}} ### Upgrading to `v26.1` and later versions diff --git a/doc/user/content/self-managed-deployments/upgrading/materialize-instances/_index.md b/doc/user/content/self-managed-deployments/upgrading/materialize-instances/_index.md new file mode 100644 index 0000000000000..6fccc4ac55a86 --- /dev/null +++ b/doc/user/content/self-managed-deployments/upgrading/materialize-instances/_index.md @@ -0,0 +1,86 @@ +--- +title: "Upgrading Materialize Instances" +description: "Upgrading Materialize instances for Self-Managed deployments." +disable_list: true +menu: + main: + parent: "upgrading" + identifier: "upgrading-materialize-instances" + weight: 20 +--- + +{{< important >}} + +When upgrading Materialize, always upgrade the Helm Chart and Materialize +Operator first. See [Upgrading the Helm Chart and Materialize Operator](/self-managed-deployments/upgrading/#upgrading-the-helm-chart-and-materialize-operator). + +{{}} + +## CRD API Versions + +Starting in v26.30, the Materialize Operator supports two CRD API versions: + +- **v1** simplifies the upgrade process. Rollouts trigger automatically when spec fields change, removing the need to manually set a `requestRollout` UUID. +- **v1alpha1** uses the original two-step upgrade process: first stage changes, then trigger a rollout with a new `requestRollout` UUID. + +Switching to v1 is **opt-in** for now. Upgrading the operator to v26.30+ does not change your existing v1alpha1 CRs or their behavior. You can continue using v1alpha1 until the next major release, switching individual instances to v1 at your own pace. + +{{< important >}} +In the next major release, all Materialize CRs will be force upgraded to v1. You will still be able to apply v1alpha1 CRs, but they will be auto-converted to v1 and use the v1 rollout behavior. We recommend opting in to v1 at your convenience to migrate on your own schedule before the upgrade is mandatory. +{{}} + +Select the instructions for your CRD API version: + +- [v1 (v26.30+)](/self-managed-deployments/upgrading/materialize-instances/v1/) +- [v1alpha1 (before v26.30)](/self-managed-deployments/upgrading/materialize-instances/v1alpha1/) + +## Switching from v1alpha1 to v1 + +Switching to v1 is opt-in and does not trigger a rollout on its own. Before switching, ensure you have completed the prerequisites in the [v26.30 upgrade notes](/self-managed-deployments/upgrading/#upgrading-to-v2630-and-later-versions) (cert-manager, network/firewall changes), and have enabled the v1 CRD by setting the Helm value `operator.args.installV1CRD=true` on the operator. Without this value, the operator only installs the v1alpha1 CRD version, and the Kubernetes API server rejects v1 CRs. + +### How it works + +The v1alpha1 CRD remains the storage version. When you submit a v1 CR, the operator's conversion webhook automatically converts it to v1alpha1 for storage. During conversion, the webhook computes a SHA256 hash of the spec and derives a deterministic `requestRollout` UUID from it. This means: + +- If the spec hasn't changed, the same UUID is generated, so **no unintended rollout is triggered** by switching API versions alone. +- If the spec has changed, a different UUID is produced, automatically triggering a rollout. + +### Using kubectl + +To switch an existing instance to v1, apply your CR with the updated `apiVersion` and remove the `requestRollout` field: + +```shell +kubectl apply -f - < + namespace: +spec: + environmentdImageRef: + backendSecretName: + # ... other spec fields (copy from your existing CR, removing requestRollout) +EOF +``` + +Or patch the API version on an existing CR: + +```shell +kubectl patch materialize \ + -n \ + --type='merge' \ + -p '{"apiVersion":"materialize.cloud/v1"}' +``` + +### Using Terraform + +If you are managing your Materialize instance with the [Materialize Terraform modules](https://github.com/MaterializeInc/materialize-terraform-self-managed), set: + +```hcl +crd_version = "v1" +request_rollout = null +``` + +### Switching back to v1alpha1 + +You can switch back to v1alpha1 at any time by reapplying your CR with `apiVersion: materialize.cloud/v1alpha1` and an explicit `requestRollout` UUID. diff --git a/doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1.md b/doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1.md new file mode 100644 index 0000000000000..a8cd458f1352f --- /dev/null +++ b/doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1.md @@ -0,0 +1,173 @@ +--- +title: "v1" +description: "Upgrading Self-Managed Materialize instances using the v1 CRD API (v26.30+)." +menu: + main: + parent: "upgrading-materialize-instances" + identifier: "upgrading-v1" + weight: 10 +--- + +{{< note >}} +If you are currently using v1alpha1 and want to switch, see [Switching from v1alpha1 to v1](/self-managed-deployments/upgrading/materialize-instances/#switching-from-v1alpha1-to-v1). +{{}} + +## Upgrading Materialize Instances + +With the `v1` CRD, rollouts trigger automatically when spec fields change. +Unlike `v1alpha1`, there is no `requestRollout` field. You only need to update the spec and the operator handles the rest. + +**After** you have upgraded your Materialize Operator, upgrade your Materialize +instance(s) to the **APP Version** of the Operator. To find the version of your +currently deployed Materialize Operator: + +```shell +helm list -n materialize +``` + +You will use the returned **App Version** for the updated `environmentdImageRef` +value. Specifically, for your Materialize instance(s), set +`environmentdImageRef` value to use the new version: + +``` +spec: + environmentdImageRef: docker.io/materialize/environmentd: +``` + +#### Using `kubectl patch` + +To initiate the Materialize instance version upgrade, you can +use the `kubectl patch` command; for example, if the **App Version** is {{< self-managed/versions/get-latest-version >}}: + +```shell +kubectl patch materialize \ + -n \ + --type='merge' \ + -p "{\"apiVersion\": \"materialize.cloud/v1\", \"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\"}}" +``` + + +#### Using YAML Definition + +Alternatively, you can update your Materialize custom resource definition directly: + +```yaml +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} # Update version as needed + forceRollout: 33333333-3333-3333-3333-333333333333 # Optional: for forced rollouts + rolloutStrategy: WaitUntilReady # The mechanism to use when rolling out the new version. Can be WaitUntilReady or ImmediatelyPromoteCausingDowntime + backendSecretName: materialize-backend +``` + +Apply the updated definition: + +```shell +kubectl apply -f materialize.yaml +``` + +#### Using Terraform + +If you are managing your Materialize instance with the +[Materialize Terraform modules](https://github.com/MaterializeInc/materialize-terraform-self-managed), +update the `environmentd_version` variable: + +```hcl +module "materialize_instance" { + source = "../kubernetes/modules/materialize-instance" + + crd_version = "v1" + request_rollout = null + environmentd_version = "{{< self-managed/versions/get-latest-version >}}" # Update to the new version + # ... other configuration +} +``` + +## Rollout Configuration + +With `v1`, the operator computes a hash of the spec fields and automatically triggers a rollout when the hash changes. The `requestRollout` field from `v1alpha1` is not used. + +### `forceRollout` + +Specify a new `UUID` value for `forceRollout` to trigger a rollout even when there are +no other changes to the instance. The `forceRollout` value is included in the +hash calculation, so changing it produces a new hash and triggers a rollout. + +```shell +kubectl patch materialize \ + -n materialize-environment \ + --type='merge' \ + -p "{\"apiVersion\": \"materialize.cloud/v1\", \"spec\": {\"forceRollout\": \"$(uuidgen)\"}}" +``` + +### Rollout strategies + +Rollout strategies control how Materialize transitions from the current generation to a new generation during an upgrade. + +The behavior of the new version rollout follows your `rolloutStrategy` setting. + +#### *WaitUntilReady* - ***Default*** + +`WaitUntilReady` creates a new generation of pods and automatically promotes +them as soon as they catch up to the old generation and become `ReadyToPromote`. +Because both generations run simultaneously until the promotion, this strategy +temporarily doubles the required resources to run Materialize. + +{{< note >}} + +While the new generation waits to become `ReadyToPromote`, it runs in a +read-only, un-promoted state and holds back compaction. To prevent it from +sitting in this state indefinitely (which can cause incident-inducing load when +it is eventually promoted), the rollout is bounded by the `rolloutRequestTimeout` +field in the Materialize spec, which defaults to `24h`. + +If the new generation does not become `ReadyToPromote` within +`rolloutRequestTimeout`, the operator cancels the rollout: the new generation is +torn down and the previously-active generation continues serving. You can then +trigger a new rollout by setting `forceRollout` to a new value. + +{{< /note >}} + +#### *ImmediatelyPromoteCausingDowntime* +{{< warning >}} Using the `ImmediatelyPromoteCausingDowntime` rollout flag will cause downtime. {{< /warning >}} + +`ImmediatelyPromoteCausingDowntime` tears down the prior generation, and immediately promotes the new generation without waiting for it to hydrate. This causes downtime until the new generation has hydrated. However, it does not require additional resources. + +#### *ManuallyPromote* + +`ManuallyPromote` allows you to choose when to promote the new generation. This means you can time the promotion for periods when load is low, minimizing the impact of potential downtime for any clients connected to Materialize. This strategy temporarily doubles the required resources to run Materialize. + +To minimize downtime, wait until the new generation has fully hydrated and caught up to the prior generation before promoting. To check hydration status, inspect the `UpToDate` condition in the Materialize resource status. When hydration completes, the condition will be `ReadyToPromote`. + +To promote, update the `forcePromote` field to match the `requestedRolloutHash` field in the Materialize status. If you need to promote before hydration completes, you can set `forcePromote` immediately, but clients may experience downtime. + +{{< warning >}} Leaving a new generation unpromoted for over 6 hours may cause downtime. {{< /warning >}} + +**Do not leave new generations unpromoted indefinitely**. They should either be promoted or canceled. New generations open a read hold on the metadata database that prevents compaction. This hold is only released when the generation is promoted or canceled. If left open too long, promoting or canceling can trigger a spike in deletion load on the metadata database, potentially causing downtime. It is not recommended to leave generations unpromoted for over 6 hours. + + +## Verifying the Upgrade + +After initiating the rollout, you can monitor the status field of the Materialize custom resource to check on the upgrade. + +```shell +# Watch the status of your Materialize environment +kubectl get materialize -n materialize-environment -w + +# Check the logs of the operator +kubectl logs -l app.kubernetes.io/name=materialize-operator -n materialize +``` + +## Cancelling the Upgrade + +You may want to cancel an in-progress rollout if the upgrade has failed. This may be indicated by new pods not being healthy. Before cancelling, verify that the upgrade has not already completed by checking that the deploy generation (found via `status.activeGeneration`) is still the one from before the upgrade. Once an upgrade has already happened, you cannot revert using this method. + +To cancel an in-progress rollout and revert to the last completed rollout state, you must revert the Materialize resource to the contents it had before triggering the rollout: + +```shell +kubectl apply -f previous_materialize_configuration.yaml +``` diff --git a/doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1alpha1.md b/doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1alpha1.md new file mode 100644 index 0000000000000..15b9b72e22cc5 --- /dev/null +++ b/doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1alpha1.md @@ -0,0 +1,234 @@ +--- +title: "v1alpha1" +description: "Upgrading Self-Managed Materialize instances using the v1alpha1 CRD API (before v26.30)." +menu: + main: + parent: "upgrading-materialize-instances" + identifier: "upgrading-v1alpha1" + weight: 20 +aliases: + - /self-managed-deployments/upgrading/v1alpha1-upgrade-instructions/ +--- + +## Upgrading Materialize Instances + +**After** you have upgraded your Materialize Operator, upgrade your Materialize +instance(s) to the **APP Version** of the Operator. To find the version of your +currently deployed Materialize Operator: + +```shell +helm list -n materialize +``` + +You will use the returned **App Version** for the updated `environmentdImageRef` +value. Specifically, for your Materialize instance(s), set +`environmentdImageRef` value to use the new version: + +``` +spec: + environmentdImageRef: docker.io/materialize/environmentd: +``` + +To minimize unexpected downtime and avoid connection drops at critical +periods for your application, the upgrade process involves two steps: + +- First, stage the changes (update the `environmentdImageRef` with the new + version) to the Materialize custom resource. The Operator watches for changes + but does not automatically roll out the changes. + +- Second, roll out the changes by specifying a new UUID for `requestRollout`. + +### Stage the Materialize instance version change + +To stage the Materialize instances version upgrade, update the +`environmentdImageRef` field in the Materialize custom resource spec to the +compatible version of your currently deployed Materialize Operator. + +To stage, but **not** rollout, the Materialize instance version upgrade, you can +use the `kubectl patch` command; for example, if the **App Version** is {{< self-managed/versions/get-latest-version >}}: + +```shell +kubectl patch materialize \ + -n \ + --type='merge' \ + -p "{\"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\"}}" +``` + +{{< note >}} +Until you specify a new `requestRollout`, the Operator watches for updates but +does not roll out the changes. +{{< /note >}} + + +### Applying the changes via `requestRollout` + +To apply the Materialize instance upgrade, you must update the `requestRollout` field in the Materialize custom resource spec to a new UUID. +Be sure to consult the [Rollout Configurations](#rollout-configuration) to ensure you've selected the correct rollout behavior. +```shell +# Then trigger the rollout with a new UUID +kubectl patch materialize \ + -n \ + --type='merge' \ + -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\"}}" +``` + +### Staging and applying in a single command + +Although separating the staging and rollout of the changes into two steps can +minimize unexpected downtime and avoid connection drops at critical periods, you +can, if preferred, combine both operations in a single command + +```shell +kubectl patch materialize \ + -n materialize-environment \ + --type='merge' \ + -p "{\"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\", \"requestRollout\": \"$(uuidgen)\"}}" +``` + +#### Using YAML Definition + +Alternatively, you can update your Materialize custom resource definition directly: + +```yaml +apiVersion: materialize.cloud/v1alpha1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} # Update version as needed + requestRollout: 22222222-2222-2222-2222-222222222222 # Use a new UUID + forceRollout: 33333333-3333-3333-3333-333333333333 # Optional: for forced rollouts + inPlaceRollout: false # In Place rollout is deprecated and ignored. Please use rolloutStrategy + rolloutStrategy: WaitUntilReady # The mechanism to use when rolling out the new version. Can be WaitUntilReady or ImmediatelyPromoteCausingDowntime + backendSecretName: materialize-backend +``` + +Apply the updated definition: + +```shell +kubectl apply -f materialize.yaml +``` + +## Rollout Configuration + +### `requestRollout` + +Specify a new `UUID` value for the `requestRollout` to roll out the changes to +the Materialize instance. + +{{< note >}} + +`requestRollout` without the `forcedRollout` field only rolls out if changes +exist to the Materialize instance. To roll out even if there are no changes to +the instance, use with `forcedRollouts`. + +{{< /note >}} + +```shell +# Only rolls out if there are changes +kubectl patch materialize \ + -n \ + --type='merge' \ + -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\"}}" +``` +#### `requestRollout` with `forcedRollouts` + +Specify a new `UUID` value for `forcedRollout` to roll out even when there are +no changes to the instance. Use `forcedRollout` with `requestRollout`. + +```shell +kubectl patch materialize \ + -n materialize-environment \ + --type='merge' \ + -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\", \"forceRollout\": \"$(uuidgen)\"}}" +``` + +### Rollout strategies + +Rollout strategies control how Materialize transitions from the current generation to a new generation during an upgrade. + +The behavior of the new version rollout follows your `rolloutStrategy` setting. + +#### *WaitUntilReady* - ***Default*** + +`WaitUntilReady` creates a new generation of pods and automatically promotes +them as soon as they catch up to the old generation and become `ReadyToPromote`. +Because both generations run simultaneously until the promotion, this strategy +temporarily doubles the required resources to run Materialize. + +{{< note >}} + +While the new generation waits to become `ReadyToPromote`, it runs in a +read-only, un-promoted state and holds back compaction. To prevent it from +sitting in this state indefinitely (which can cause incident-inducing load when +it is eventually promoted), the rollout is bounded by the `rolloutRequestTimeout` +field in the Materialize spec, which defaults to `24h`. + +If the new generation does not become `ReadyToPromote` within +`rolloutRequestTimeout`, the operator cancels the rollout: the new generation is +torn down and the previously-active generation continues serving. You can then +trigger a new rollout by setting `requestRollout` to a new value. + +{{< /note >}} + +#### *ImmediatelyPromoteCausingDowntime* +{{< warning >}} Using the `ImmediatelyPromoteCausingDowntime` rollout flag will cause downtime. {{< /warning >}} + +`ImmediatelyPromoteCausingDowntime` tears down the prior generation, and immediately promotes the new generation without waiting for it to hydrate. This causes downtime until the new generation has hydrated. However, it does not require additional resources. + +#### *ManuallyPromote* + +`ManuallyPromote` allows you to choose when to promote the new generation. This means you can time the promotion for periods when load is low, minimizing the impact of potential downtime for any clients connected to Materialize. This strategy temporarily doubles the required resources to run Materialize. + +To minimize downtime, wait until the new generation has fully hydrated and caught up to the prior generation before promoting. To check hydration status, inspect the `UpToDate` condition in the Materialize resource status. When hydration completes, the condition will be `ReadyToPromote`. + +To promote, update the `forcePromote` field to match the `requestRollout` field in the Materialize spec. If you need to promote before hydration completes, you can set `forcePromote` immediately, but clients may experience downtime. + +{{< warning >}} Leaving a new generation unpromoted for over 6 hours may cause downtime. {{< /warning >}} + +**Do not leave new generations unpromoted indefinitely**. They should either be promoted or canceled. New generations open a read hold on the metadata database that prevents compaction. This hold is only released when the generation is promoted or canceled. If left open too long, promoting or canceling can trigger a spike in deletion load on the metadata database, potentially causing downtime. It is not recommended to leave generations unpromoted for over 6 hours. + +#### *inPlaceRollout* - ***Deprecated*** + +The setting is ignored. + +## Verifying the Upgrade + +After initiating the rollout, you can monitor the status field of the Materialize custom resource to check on the upgrade. + +```shell +# Watch the status of your Materialize environment +kubectl get materialize -n materialize-environment -w + +# Check the logs of the operator +kubectl logs -l app.kubernetes.io/name=materialize-operator -n materialize +``` + +## Cancelling the Upgrade + +You may want to cancel an in-progress rollout if the upgrade has failed. This may be indicated by new pods not being healthy. Before cancelling, verify that the upgrade has not already completed by checking that the deploy generation (found via `status.activeGeneration`) is still the one from before the upgrade. Once an upgrade has already happened, you cannot revert using this method. + +To cancel an in-progress rollout and revert to the last completed rollout +state, revert both `requestRollout` and `environmentdImageRef` back to the +values from the last completed rollout. Reverting `environmentdImageRef` +alongside `requestRollout` keeps the spec aligned with what is actually +running, so a later rollout doesn't accidentally pick up the previously +attempted upgrade image. + +First, retrieve the last completed rollout request ID and the matching +environmentd image ref from your Materialize CR: + +```shell +kubectl get materialize -n materialize-environment \ + -o jsonpath='{.status.lastCompletedRolloutRequest} {.status.lastCompletedRolloutEnvironmentdImageRef}' +``` + +Then, set both fields back to these values in a single patch: + +```shell +kubectl patch materialize \ + -n materialize-environment \ + --type='merge' \ + -p "{\"spec\": {\"requestRollout\": \"\", \"environmentdImageRef\": \"\"}}" +``` diff --git a/doc/user/data/self_managed/materialize_crd_descriptions_v1.json b/doc/user/data/self_managed/materialize_crd_descriptions_v1.json new file mode 100644 index 0000000000000..6085d5ce6b00e --- /dev/null +++ b/doc/user/data/self_managed/materialize_crd_descriptions_v1.json @@ -0,0 +1,558 @@ +[ + [ + "MaterializeSpec", + [ + { + "name": "backendSecretName", + "type": "String", + "description": "The name of a secret containing `metadata_backend_url` and `persist_backend_url`.\nIt may also contain `external_login_password_mz_system`, which will be used as\nthe password for the `mz_system` user if `authenticatorKind` is `Password`.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "environmentdImageRef", + "type": "String", + "description": "The environmentd image to run.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "authenticatorKind", + "type": "Enum", + "description": "How to authenticate with Materialize.\n\nValid values:\n- `Frontegg`:
Authenticate users using Frontegg.\n- `Password`:
Authenticate users using internally stored password hashes.\n The backend secret must contain external_login_password_mz_system.\n- `Sasl`:
Authenticate users using SASL.\n- `Oidc`:
Authenticate users using OIDC (JWT tokens).\n- `None` (default):
Do not authenticate users. Trust they are who they say they are without verification.", + "default": "None", + "required": false, + "deprecated": false + }, + { + "name": "balancerdExternalCertificateSpec", + "type": "MaterializeCertSpec", + "description": "The configuration for generating an x509 certificate using cert-manager for balancerd\nto present to incoming connections.\nThe `dnsNames` and `issuerRef` fields are required.\n\nThis field is excluded from the rollout hash and changes will not trigger a rollout.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "balancerdReplicas", + "type": "Integer", + "description": "Number of balancerd pods to create.\n\nThis field is excluded from the rollout hash and changes will not trigger a rollout.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "balancerdResourceRequirements", + "type": "io.k8s.api.core.v1.ResourceRequirements", + "description": "Resource requirements for the balancerd pod.\n\nThis field is excluded from the rollout hash and changes will not trigger a rollout.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "consoleExternalCertificateSpec", + "type": "MaterializeCertSpec", + "description": "The configuration for generating an x509 certificate using cert-manager for the console\nto present to incoming connections.\nThe `dnsNames` and `issuerRef` fields are required.\nNot yet implemented.\n\nThis field is excluded from the rollout hash and changes will not trigger a rollout.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "consoleReplicas", + "type": "Integer", + "description": "Number of console pods to create.\n\nThis field is excluded from the rollout hash and changes will not trigger a rollout.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "consoleResourceRequirements", + "type": "io.k8s.api.core.v1.ResourceRequirements", + "description": "Resource requirements for the console pod.\n\nThis field is excluded from the rollout hash and changes will not trigger a rollout.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "enableRbac", + "type": "Bool", + "description": "Whether to enable role based access control. Defaults to false.", + "default": false, + "required": false, + "deprecated": false + }, + { + "name": "environmentId", + "type": "Uuid", + "description": "The value used by environmentd (via the --environment-id flag) to\nuniquely identify this instance. Must be globally unique, and\nis required if a license key is not provided.\nNOTE: This value MUST NOT be changed in an existing instance,\nsince it affects things like the way data is stored in the persist\nbackend.", + "default": "00000000-0000-0000-0000-000000000000", + "required": false, + "deprecated": false + }, + { + "name": "environmentdConnectionRoleArn", + "type": "String", + "description": "If running in AWS, override the IAM role to use to support\nthe CREATE CONNECTION feature.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "environmentdExtraArgs", + "type": "Array", + "description": "Extra args to pass to the environmentd binary.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "environmentdExtraEnv", + "type": "Array", + "description": "Extra environment variables to pass to the environmentd binary.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "environmentdResourceRequirements", + "type": "io.k8s.api.core.v1.ResourceRequirements", + "description": "Resource requirements for the environmentd pod.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "environmentdScratchVolumeStorageRequirement", + "type": "io.k8s.apimachinery.pkg.api.resource.Quantity", + "description": "Amount of disk to allocate, if a storage class is provided.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "forcePromote", + "type": "String", + "description": "If `forcePromote` is set to the same value as the `status.requestedRolloutHash`,\ncurrent rollout will skip waiting for clusters in the new\ngeneration to rehydrate before promoting the new environmentd to\nleader.\n\nThis field is excluded from the rollout hash and changes will not trigger a rollout.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "forceRollout", + "type": "Uuid", + "description": "This value will force the controller to detect the spec as changed\neven if no other changes happened. This can be used to force a rollout\nto a new generation even without making any meaningful changes.", + "default": "00000000-0000-0000-0000-000000000000", + "required": false, + "deprecated": false + }, + { + "name": "internalCertificateSpec", + "type": "MaterializeCertSpec", + "description": "The cert-manager Issuer or ClusterIssuer to use for database internal communication.\nThe `issuerRef` field is required.\nThis currently is only used for environmentd, but will eventually support clusterd.\nNot yet implemented.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "podAnnotations", + "type": "Map", + "description": "Annotations to apply to the pods.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "podLabels", + "type": "Map", + "description": "Labels to apply to the pods.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "rolloutRequestTimeout", + "type": "RolloutRequestTimeout", + "description": "The maximum amount of time a rollout may remain in progress before\nit is automatically cancelled.\n\nWhile a rollout is in progress, the new generation of `environmentd`\nruns in a read-only, un-promoted state and holds back compaction via\nread holds. Leaving it in this state for too long can cause\nincident-inducing load when it is eventually promoted, so the\noperator cancels the rollout once this timeout is exceeded: the new\ngeneration is torn down and the previously-active generation\ncontinues serving. A new rollout can then be triggered by setting\n`forceRollout` to a new value.\n\nThis does not apply to the `ImmediatelyPromoteCausingDowntime`\nrollout strategy or to force-promoted rollouts, since by the time\nthose are in progress the old generation may already be gone.\n\nThe value is parsed as a human-readable duration, e.g. `24h`,\n`90m`, or `1h 30m`. Defaults to [`DEFAULT_ROLLOUT_REQUEST_TIMEOUT`]\nwhen omitted (the API server fills it in); an unparseable value also\nfalls back to that default.", + "default": "24h", + "required": false, + "deprecated": false + }, + { + "name": "rolloutStrategy", + "type": "Enum", + "description": "Rollout strategy to use when upgrading this Materialize instance.\n\nValid values:\n- `WaitUntilReady` (default):
Create a new generation of pods, leaving the old generation around until the\n new ones are ready to take over.\n This minimizes downtime, and is what almost everyone should use.\n- `ManuallyPromote`:
Create a new generation of pods, leaving the old generation as the serving generation\n until the user manually promotes the new generation.\n \n When using `ManuallyPromote`, the new generation can be promoted at any\n time, even if it has dataflows that are not fully caught up, by setting\n `forcePromote` to the same value as `requestRollout` in the Materialize spec.\n \n To minimize downtime, promotion should occur when the new generation\n has caught up to the prior generation. To determine if the new\n generation has caught up, consult the `UpToDate` condition in the\n status of the Materialize Resource. If the condition's reason is\n `ReadyToPromote` the new generation is ready to promote.\n \n {{}}\n Do not leave new generations unpromoted indefinitely.\n \n The new generation keeps open read holds which prevent compaction. Once promoted or\n cancelled, those read holds are released. If left unpromoted for an extended time, this\n data can build up, and can cause extreme deletion load on the metadata backend database\n when finally promoted or cancelled.\n \n To guard against this, a rollout that remains in progress longer\n than `rolloutRequestTimeout` (default 24h) is automatically\n cancelled.\n {{}}\n- `ImmediatelyPromoteCausingDowntime`:
{{}}\n THIS WILL CAUSE YOUR MATERIALIZE INSTANCE TO BE UNAVAILABLE FOR SOME TIME!!!\n \n This strategy should ONLY be used by customers with physical hardware who do not have\n enough hardware for the `WaitUntilReady` strategy. If you think you want this, please\n consult with Materialize engineering to discuss your situation.\n {{}}\n \n Tear down the old generation of pods and promote the new generation of pods immediately,\n without waiting for the new generation of pods to be ready.", + "default": "WaitUntilReady", + "required": false, + "deprecated": false + }, + { + "name": "serviceAccountAnnotations", + "type": "Map", + "description": "Annotations to apply to the service account.\n\nAnnotations on service accounts are commonly used by cloud providers for IAM.\nAWS uses \"eks.amazonaws.com/role-arn\".\nAzure uses \"azure.workload.identity/client-id\", but\nadditionally requires \"azure.workload.identity/use\": \"true\" on the pods.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "serviceAccountLabels", + "type": "Map", + "description": "Labels to apply to the service account.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "serviceAccountName", + "type": "String", + "description": "Name of the kubernetes service account to use.\nIf not set, we will create one with the same name as this Materialize object.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "systemParameterConfigmapName", + "type": "String", + "description": "The name of a ConfigMap containing system parameters in JSON format.\nThe ConfigMap must contain a `system-params.json` key whose value\nis a valid JSON object containing valid system parameters.\n\nRun `SHOW ALL` in SQL to see a subset of configurable system parameters.\n\nExample ConfigMap:\n```yaml\ndata:\n system-params.json: |\n {\n \"max_connections\": 1000\n }\n```", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "MaterializeCertSpec", + [ + { + "name": "dnsNames", + "type": "Array", + "description": "Additional DNS names the certificate will be valid for.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "duration", + "type": "String", + "description": "Duration the certificate will be requested for.\nValue must be in units accepted by Go\n[`time.ParseDuration`](https://golang.org/pkg/time/#ParseDuration).", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "issuerRef", + "type": "CertificateIssuerRef", + "description": "Reference to an `Issuer` or `ClusterIssuer` that will generate the certificate.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "privateKeyAlgorithm", + "type": "CertificatePrivateKeyAlgorithm", + "description": "Optional algorithm to use for the private key. If not specified, a recommended default will be chosen.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "privateKeySize", + "type": "Integer", + "description": "Optional size for the private key.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "renewBefore", + "type": "String", + "description": "Duration before expiration the certificate will be renewed.\nValue must be in units accepted by Go\n[`time.ParseDuration`](https://golang.org/pkg/time/#ParseDuration).", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "secretTemplate", + "type": "CertificateSecretTemplate", + "description": "Additional annotations and labels to include in the Certificate object.", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "CertificateSecretTemplate", + [ + { + "name": "annotations", + "type": "Map", + "description": "Annotations is a key value map to be copied to the target Kubernetes Secret.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "labels", + "type": "Map", + "description": "Labels is a key value map to be copied to the target Kubernetes Secret.", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "CertificateIssuerRef", + [ + { + "name": "name", + "type": "String", + "description": "Name of the resource being referred to.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "group", + "type": "String", + "description": "Group of the resource being referred to.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "kind", + "type": "String", + "description": "Kind of the resource being referred to.", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.ResourceRequirements", + [ + { + "name": "claims", + "type": "Array", + "description": "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container.\n\nThis is an alpha field and requires enabling the DynamicResourceAllocation feature gate.\n\nThis field is immutable. It can only be set for containers.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "limits", + "type": "Map", + "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "requests", + "type": "Map", + "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.ResourceClaim", + [ + { + "name": "name", + "type": "String", + "description": "Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "request", + "type": "String", + "description": "Request is the name chosen for a request in the referenced claim. If empty, everything from the claim is made available, otherwise only the result of this request.", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.EnvVar", + [ + { + "name": "name", + "type": "String", + "description": "Name of the environment variable. Must be a C_IDENTIFIER.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "value", + "type": "String", + "description": "Variable references $(VAR_NAME) are expanded using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "valueFrom", + "type": "io.k8s.api.core.v1.EnvVarSource", + "description": "Source for the environment variable's value. Cannot be used if value is not empty.", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.EnvVarSource", + [ + { + "name": "configMapKeyRef", + "type": "io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "fieldRef", + "type": "io.k8s.api.core.v1.ObjectFieldSelector", + "description": "Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "resourceFieldRef", + "type": "io.k8s.api.core.v1.ResourceFieldSelector", + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "secretKeyRef", + "type": "io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.SecretKeySelector", + [ + { + "name": "key", + "type": "String", + "description": "The key of the secret to select from. Must be a valid secret key.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "name", + "type": "String", + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "optional", + "type": "Bool", + "description": "Specify whether the Secret or its key must be defined", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.ResourceFieldSelector", + [ + { + "name": "resource", + "type": "String", + "description": "Required: resource to select", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "containerName", + "type": "String", + "description": "Container name: required for volumes, optional for env vars", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "divisor", + "type": "io.k8s.apimachinery.pkg.api.resource.Quantity", + "description": "Specifies the output format of the exposed resources, defaults to \"1\"", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.ObjectFieldSelector", + [ + { + "name": "fieldPath", + "type": "String", + "description": "Path of the field to select in the specified API version.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "apiVersion", + "type": "String", + "description": "Version of the schema the FieldPath is written in terms of, defaults to \"v1\".", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.ConfigMapKeySelector", + [ + { + "name": "key", + "type": "String", + "description": "The key to select.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "name", + "type": "String", + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "optional", + "type": "Bool", + "description": "Specify whether the ConfigMap or its key must be defined", + "default": null, + "required": false, + "deprecated": false + } + ] + ] +] diff --git a/doc/user/data/self_managed/materialize_crd_descriptions.json b/doc/user/data/self_managed/materialize_crd_descriptions_v1alpha1.json similarity index 100% rename from doc/user/data/self_managed/materialize_crd_descriptions.json rename to doc/user/data/self_managed/materialize_crd_descriptions_v1alpha1.json diff --git a/doc/user/data/self_managed/upgrades.yml b/doc/user/data/self_managed/upgrades.yml index d53015f6ee8d2..e36f6aeba6af5 100644 --- a/doc/user/data/self_managed/upgrades.yml +++ b/doc/user/data/self_managed/upgrades.yml @@ -152,7 +152,7 @@ The **APP VERSION** will be the value that you will use for upgrading Materialize instances. -- name: upgrade-materialize-instance +- name: upgrade-materialize-instance-v1alpha1 content: | **After** you have upgraded your Materialize Operator, upgrade your @@ -166,8 +166,8 @@ - Second, rolling out the changes via a `requestRollout` flag. 1. Find the name of the Materialize instance to upgrade. The sample example - deployment using the unified Terraform module deploys the Materialie - instance in the`materialize-environment` namespace. + deployment using the unified Terraform module deploys the Materialize + instance in the `materialize-environment` namespace. ```shell kubectl get materialize -n materialize-environment @@ -183,7 +183,7 @@ 1. Stage, but not rollout, the Materialize instance version upgrade. ```shell - kubectl patch materialize main\ + kubectl patch materialize main \ -n materialize-environment \ --type='merge' \ -p "{\"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\"}}" @@ -203,3 +203,39 @@ ```bash kubectl -n materialize-environment describe pod -l app=environmentd ``` + +- name: upgrade-materialize-instance + content: | + + **After** you have upgraded your Materialize Operator, upgrade your + Materialize instance(s) to the **APP Version** of the Operator. + + 1. Find the name of the Materialize instance to upgrade. The sample example + deployment using the unified Terraform module deploys the Materialize + instance in the `materialize-environment` namespace. + + ```shell + kubectl get materialize -n materialize-environment + ``` + + In the example deployment, the name of the instance is `main`. + + ```none + NAME + main + ``` + + 1. Apply the Materialize instance version upgrade. + + ```shell + kubectl patch materialize main \ + -n materialize-environment \ + --type='merge' \ + -p "{\"apiVersion\": \"materialize.cloud/v1\", \"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\"}}" + ``` + + 1. Verify the upgrade by checking the `environmentd` events: + + ```bash + kubectl -n materialize-environment describe pod -l app=environmentd + ``` diff --git a/doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions.html b/doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions-v1.html similarity index 89% rename from doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions.html rename to doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions-v1.html index 5c09e0cb47f88..66b194694e322 100644 --- a/doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions.html +++ b/doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions-v1.html @@ -1,10 +1,10 @@ {{ $types := dict }} -{{ range $.Site.Data.self_managed.materialize_crd_descriptions }} +{{ range $.Site.Data.self_managed.materialize_crd_descriptions_v1 }} {{ $types = merge $types (dict (index . 0) true) }} {{ end }} -{{ range $.Site.Data.self_managed.materialize_crd_descriptions }} +{{ range $.Site.Data.self_managed.materialize_crd_descriptions_v1 }} #### {{ index . 0 }} diff --git a/doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions-v1alpha1.html b/doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions-v1alpha1.html new file mode 100644 index 0000000000000..de77f8a4e1a95 --- /dev/null +++ b/doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions-v1alpha1.html @@ -0,0 +1,57 @@ + +{{ $types := dict }} +{{ range $.Site.Data.self_managed.materialize_crd_descriptions_v1alpha1 }} +{{ $types = merge $types (dict (index . 0) true) }} +{{ end }} + +{{ range $.Site.Data.self_managed.materialize_crd_descriptions_v1alpha1 }} +#### {{ index . 0 }} +
+ + + + + + + + +{{- range (index . 1) }} + {{- if not .deprecated }} + + + + + + {{- end }} +{{- end }} + +
Field NameRequiredDescription
{{ .name }}{{ if .required }}✅{{ end }} + + {{- $matches := findRESubmatch `(Array<|Map]+)(>*)` .type 1 -}} + {{- $captures := index $matches 0 -}} + + {{- $typePrefix := index $captures 1 -}} + {{- $baseType := index $captures 2 -}} + {{- $typeSuffix := index $captures 3 -}} + + {{- $typePrefix -}} + {{- if (index $types $baseType) -}} + + {{- end -}} + {{ $baseType }} + {{- if (index $types $baseType) -}} + + {{- end -}} + {{- $typeSuffix -}} + + + +

{{- .description | markdownify -}}

+ + + {{- if .default -}} +

Default: {{ .default }}

+ {{- end -}} + +
+{{ end }} diff --git a/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html b/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html index 94ee296c4aac3..d9b25385c5ac9 100644 --- a/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html +++ b/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html @@ -63,6 +63,33 @@ 1. Create a new `upgrade-materialize.yaml` file, updating the following fields: +
+
    +
    +
    + + | Field | Description | + |-------|-------------| + | `environmentdImageRef` | Update the version to the new version. This should be the same as the operator version: `{{ $operator_version }}`. | + | `forceRollout`| Enter a new UUID. Can be generated with `uuidgen`.
    • `forceRollout` triggers a rollout even if no other changes exist.
    | + + + ```yaml + apiVersion: materialize.cloud/v1 + kind: Materialize + metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment + spec: + environmentdImageRef: materialize/environmentd:{{ $environmentd_version }} # Update version + # forceRollout: 33333333-3333-3333-3333-333333333333 # For forced rollouts + rolloutStrategy: WaitUntilReady # The mechanism to use when rolling out the new version. + backendSecretName: materialize-backend + ``` + +
    +
    + | Field | Description | |-------|-------------| | `environmentdImageRef` | Update the version to the new version. This should be the same as the operator version: `{{ $operator_version }}`. | @@ -83,6 +110,10 @@ backendSecretName: materialize-backend ``` +
    +
    +
    + 1. Apply the upgrade-materialize.yaml file to your Materialize instance: ```shell diff --git a/doc/user/shared-content/self-managed/upgrade-notes/v26.0.md b/doc/user/shared-content/self-managed/upgrade-notes/v26.0.md index a59c43a810a08..a740e3c7e9224 100644 --- a/doc/user/shared-content/self-managed/upgrade-notes/v26.0.md +++ b/doc/user/shared-content/self-managed/upgrade-notes/v26.0.md @@ -8,7 +8,7 @@ - `ImmediatelyPromoteCausingDowntime` For more information, see - [`rolloutStrategy`](/self-managed-deployments/upgrading/#rollout-strategies). + [`rolloutStrategy`](/self-managed-deployments/upgrading/materialize-instances/v1/#rollout-strategies). - New requirements were introduced for [license keys](/releases/#license-key). To upgrade, you will first need to add a license key to the `backendSecret` diff --git a/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md b/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md new file mode 100644 index 0000000000000..eda81b9f67213 --- /dev/null +++ b/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md @@ -0,0 +1,287 @@ +- v26.30.0 introduces a new version of the Materialize CRD, `v1`. The new CRD simplifies rollouts by automatically detecting changes. This means you will no longer need to manually rotate a UUID to trigger a rollout. + +Before, on `v1alpha1`: +```yaml +apiVersion: materialize.cloud/v1alpha1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:v26.16.0 + requestRollout: 22222222-2222-2222-2222-222222222222 # ← MUST set a new UUID every upgrade +# forceRollout: 33333333-3333-3333-3333-333333333333 # ← for forced rollouts + rolloutStrategy: WaitUntilReady + backendSecretName: materialize-backend +``` + +After, on `v1`: +```yaml +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:v26.16.0 # ← just change this +# forceRollout: 33333333-3333-3333-3333-333333333333 # ← only for forced rollouts + rolloutStrategy: WaitUntilReady + backendSecretName: materialize-backend +``` + +With the new change, the `requestRollout` field will be removed, along with all previously deprecated fields. + +Switching to `v1` is **opt-in**. You may continue to apply `v1alpha1` CRs and your existing instances will behave exactly as before. We recommend you opt-in to `v1` at your convenience, as we will migrate to the `v1` behavior in the next major release. For step-by-step migration instructions, see [Switching from v1alpha1 to v1](/self-managed-deployments/upgrading/materialize-instances/#switching-from-v1alpha1-to-v1). + + +{{< important >}} + +Using the `v1` CRD requires infrastructure changes and an additional Helm +value, `operator.args.installV1CRD=true`. If you continue to use `v1alpha1`, +no infrastructure changes are required, but we recommend making them anyway, +as TLS certificates will become required in a future version of Materialize. + +{{< /important >}} + +Materialize uses conversion webhooks to allow you to gracefully migrate from +`v1alpha1` to `v1`. The `v1` CRD and its conversion webhook are only installed +when you set the Helm value `operator.args.installV1CRD=true`. Before enabling +it, you need to install cert-manager (or provide your own certificate) and +allow internal network ingress on port `8001`. + +Choose the tab that matches your deployment method: + +
    +
      +
      +
      + +If you are using the [supported Terraform +modules](https://github.com/MaterializeInc/materialize-terraform-self-managed), +the required infrastructure changes (cert-manager and network ingress) will be +handled for you automatically. + +To upgrade, update each module's `source` to point to the new release tag and +run `terraform init -upgrade && terraform plan && terraform apply`. To enable +the `v1` CRD, also set the Helm value `operator.args.installV1CRD=true` in the +values passed to the operator module. + +The key modules and their dependency chain are shown below. Your configuration +may include additional modules (networking, storage, database, node pools, etc.) +— update those to the same release tag as well. + +
      +
        +
        +
        + +```hcl +module "eks" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//aws/modules/eks?ref=" + # ... your existing configuration ... +} + +module "cert_manager" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/cert-manager?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.eks] +} + +module "operator" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//aws/modules/operator?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.cert_manager] +} + +module "materialize_instance" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/materialize-instance?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.operator] +} +``` + +For a complete example, see +[`aws/examples/simple/main.tf`](https://github.com/MaterializeInc/materialize-terraform-self-managed/blob/main/aws/examples/simple/main.tf). + +
        +
        + +```hcl +module "gke" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//gcp/modules/gke?ref=" + # ... your existing configuration ... +} + +module "cert_manager" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/cert-manager?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.gke] +} + +module "operator" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//gcp/modules/operator?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.cert_manager] +} + +module "materialize_instance" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/materialize-instance?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.operator] +} +``` + +For a complete example, see +[`gcp/examples/simple/main.tf`](https://github.com/MaterializeInc/materialize-terraform-self-managed/blob/main/gcp/examples/simple/main.tf). + +
        +
        + +```hcl +module "aks" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//azure/modules/aks?ref=" + # ... your existing configuration ... +} + +module "cert_manager" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/cert-manager?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.aks] +} + +module "operator" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//azure/modules/operator?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.cert_manager] +} + +module "materialize_instance" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/materialize-instance?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.operator] +} +``` + +For a complete example, see +[`azure/examples/simple/main.tf`](https://github.com/MaterializeInc/materialize-terraform-self-managed/blob/main/azure/examples/simple/main.tf). + +
        +
        +
        + +
        +
        + +If you are using the legacy Terraform modules +([AWS](https://github.com/MaterializeInc/terraform-aws-materialize), +[GCP](https://github.com/MaterializeInc/terraform-gcp-materialize), or +[Azure](https://github.com/MaterializeInc/terraform-azure-materialize)), +we recommend migrating to the [new supported Terraform +modules](https://github.com/MaterializeInc/materialize-terraform-self-managed) +before opting in to the `v1` CRD. + +The new modules include built-in support for the conversion webhooks used by +the `v1` CRD, including cert-manager installation and network policy +configuration. The legacy modules do not include these changes, so you would +need to apply them manually (see the **Manual** tab). + +For migration guidance, see the documentation for your cloud provider: + +- [AWS migration guide](https://github.com/MaterializeInc/materialize-terraform-self-managed/tree/main/aws/examples/migration) +- [GCP migration guide](https://github.com/MaterializeInc/materialize-terraform-self-managed/tree/main/gcp/examples/migration) +- [Azure migration guide](https://github.com/MaterializeInc/materialize-terraform-self-managed/tree/main/azure/examples/migration) + +
        +
        + +If you are not using our Terraform modules, you **must** complete the following +steps before enabling the `v1` CRD: + +**1. Install cert-manager** + +The conversion webhook requires a TLS certificate. +The Helm chart defaults to using [cert-manager](https://cert-manager.io/) +to automatically create and manage this certificate. cert-manager must be +installed **before** enabling the `v1` CRD. + +If you prefer to provide your own certificate instead of using cert-manager, +set the following Helm values: +- `operator.certificate.source`: `secret` +- `operator.certificate.secretName`: the name of the Kubernetes Secret + containing `ca.crt`, `tls.crt`, and `tls.key` entries. + +**2. Allow network access to the webhook port** + +The conversion webhooks require the Kubernetes API server to reach the +`orchestratord` pod on port `8001`. If your cluster enforces network policies +or cloud-level firewall rules, you must allow ingress traffic on TCP port +`8001` from the API server to pods with the label +`app.kubernetes.io/name: materialize-operator`. + +**Kubernetes NetworkPolicy:** Add a policy that allows ingress from the +Kubernetes API server on port `8001` to the `materialize-operator` pods in the +namespace where the operator is deployed: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-api-server-ingress-to-conversion-webhook + namespace: materialize # the namespace where the operator runs +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: materialize-operator + policyTypes: + - Ingress + ingress: + - ports: + - protocol: TCP + port: 8001 +``` + +**Cloud firewall rules (e.g., AWS security groups, GCP firewall rules):** +Ensure the node security group or firewall allows inbound TCP traffic on +port `8001` from the Kubernetes control plane. For example, on AWS, add an +ingress rule to the EKS node security group allowing port `8001` from the +cluster security group. On GCP with private clusters, add a firewall rule +allowing port `8001` from the GKE control plane CIDR. + +For a complete example of the required changes across AWS, Azure, and GCP, +see [this pull request](https://github.com/MaterializeInc/materialize-terraform-self-managed/pull/160). + +**3. Enable the v1 CRD** + +Once the prerequisites above are in place, set the following Helm value when +installing or upgrading the operator: + +```yaml +operator: + args: + installV1CRD: true +``` + +This installs the `v1` version of the Materialize CRD and the conversion +webhook that converts between `v1` and `v1alpha1`. + +
        +
        +
        diff --git a/src/cloud-resources/src/bin/crd_writer.rs b/src/cloud-resources/src/bin/crd_writer.rs index 7a51c8386ba4e..b2355d5656751 100644 --- a/src/cloud-resources/src/bin/crd_writer.rs +++ b/src/cloud-resources/src/bin/crd_writer.rs @@ -8,9 +8,11 @@ // by the Apache License, Version 2.0. use std::cmp::Ordering; +use std::env; use indexmap::{IndexMap, IndexSet}; -use mz_cloud_resources::crd::materialize::v1alpha1::MaterializeSpec; +use mz_cloud_resources::crd::materialize::v1; +use mz_cloud_resources::crd::materialize::v1alpha1; use schemars::schema_for; use serde::Serialize; @@ -328,7 +330,14 @@ fn format_enum_variants_from_json( } fn main() { - let root_schema = schema_for!(MaterializeSpec); + let args: Vec = env::args().collect(); + let version = args.get(1).expect("usage: crd-writer "); + + let root_schema = match version.as_str() { + "v1alpha1" => schema_for!(v1alpha1::MaterializeSpec), + "v1" => schema_for!(v1::MaterializeSpec), + other => panic!("unknown version: {other}, expected v1alpha1 or v1"), + }; // Convert all to JSON for easier merging let schema_json = root_schema.to_value(); diff --git a/src/cloud-resources/src/crd/materialize.rs b/src/cloud-resources/src/crd/materialize.rs index afe22d675accd..483ca37f8b915 100644 --- a/src/cloud-resources/src/crd/materialize.rs +++ b/src/cloud-resources/src/crd/materialize.rs @@ -970,7 +970,7 @@ pub mod v1 { /// operator cancels the rollout once this timeout is exceeded: the new /// generation is torn down and the previously-active generation /// continues serving. A new rollout can then be triggered by setting - /// `requestRollout` to a new value. + /// `forceRollout` to a new value. /// /// This does not apply to the `ImmediatelyPromoteCausingDowntime` /// rollout strategy or to force-promoted rollouts, since by the time From 1206c36b064074ccb11c517c35576e916d0ccf21 Mon Sep 17 00:00:00 2001 From: Alex Hunt Date: Wed, 24 Jun 2026 15:28:02 +0100 Subject: [PATCH 2/6] docs: address v1 CRD review feedback and consolidate upgrade instructions Addresses the latest review round (Val, Kay, def-) on #35508. Correctness fixes: - Rollout hash covers a subset of spec fields (not the whole spec); document which fields are included/excluded (balancerd/console reconcile in place). - Note that adopting v1 on an existing v1alpha1 instance typically triggers one rollout (the hash-derived requestRollout differs from the prior hand-set one). - Fix ManuallyPromote/forcePromote guidance: forcePromote matches status.requestedRolloutHash in v1, requestRollout in v1alpha1. Fixed in the shared MaterializeRolloutStrategy doc comment and regenerated the CRD description JSON for both versions. - Drop the kubectl-patch-apiVersion "switch" example (a no-op; the stored version stays v1alpha1) in favor of full applies. - Use `terraform init` instead of `terraform init -upgrade` to avoid bumping unrelated providers/modules. Restructure: - Consolidate the per-version Upgrading Materialize Instances pages back onto the Upgrading landing page (concepts: adopting v1, rollout configuration, rollout strategies, verifying, cancelling) and into the per-environment guides (Kind, AWS, Azure, GCP), matching main's structure. Delete the v1/v1alpha1 instance subpages and repoint all inbound links to the landing anchors. Consistency: - Reorder/relabel all v1alpha1/v1 tabs (v1alpha1 first, no version parentheses) and factor the repeated blurb into a shared partial/shortcode. - Add optional v1-enablement tabs to the Kind install and the operator-upgrade step (Kind + cloud guides); make cert-manager a regular Kind install step. - Bump hardcoded image versions to the version shortcode; rename the "switching" section to "Adopting the v1 CRD". Co-Authored-By: Claude Opus 4.8 --- doc/user/content/releases/_index.md | 4 +- .../security/self-managed/authentication.md | 34 ++- doc/user/content/security/self-managed/sso.md | 63 +++- .../self-managed-deployments/_index.md | 28 +- .../configuration-system-parameters.md | 34 ++- .../installation/install-on-local-kind.md | 10 +- .../_index.md | 2 +- .../materialize-crd-field-descriptions/v1.md | 7 + .../v1alpha1.md | 7 + .../upgrading/_index.md | 284 +++++++++++++++++- .../upgrading/materialize-instances/_index.md | 86 ------ .../upgrading/materialize-instances/v1.md | 173 ----------- .../materialize-instances/v1alpha1.md | 234 --------------- .../materialize_crd_descriptions_v1.json | 2 +- ...materialize_crd_descriptions_v1alpha1.json | 2 +- doc/user/data/self_managed/upgrades.yml | 61 +++- .../self-managed/crd-version-note.html | 13 + .../self-managed/crd-version-note.html | 1 + ...stall-helm-version-local-kind-install.html | 27 ++ .../upgrade/upgrade-steps-local-kind.html | 52 +++- .../self-managed/upgrade-notes/v26.0.md | 2 +- .../self-managed/upgrade-notes/v26.30.md | 4 +- src/cloud-resources/src/crd/materialize.rs | 4 +- 23 files changed, 571 insertions(+), 563 deletions(-) delete mode 100644 doc/user/content/self-managed-deployments/upgrading/materialize-instances/_index.md delete mode 100644 doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1.md delete mode 100644 doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1alpha1.md create mode 100644 doc/user/layouts/partials/self-managed/crd-version-note.html create mode 100644 doc/user/layouts/shortcodes/self-managed/crd-version-note.html diff --git a/doc/user/content/releases/_index.md b/doc/user/content/releases/_index.md index dae05f381aca0..bb9953890e8d9 100644 --- a/doc/user/content/releases/_index.md +++ b/doc/user/content/releases/_index.md @@ -1204,7 +1204,7 @@ v26.4.0 introduces several performance improvements and bugfixes. - **Up to 3x faster hydration times for large PostgreSQL tables**: We've reduced the overhead incurred by communication between multiple *workers* on a large cluster. We've observed up to 3x throughput improvement when ingesting 1 TB PostgreSQL tables on large clusters. - **More efficient source ingestion batching**: Sources now batch writes more effectively. This can result in improved freshness and lower resource utilization, especially when a source is doing a large number of writes. - **CloudSQL HA failover support** (*Materialize Self-Managed only*): Materialize Self-Managed now offers better support for handling failovers in CloudSQL HA sources, without downtime. [Contact our support team](/support/) to enable this in your environment. -- **Manual Promotion** (*Materialize Self-Managed only*): [Rollout strategies](/self-managed-deployments/upgrading/materialize-instances/v1/#rollout-strategies) allow you control how Materialize transitions from the current generation to a new generation during an upgrade. We've added a new rollout strategy called `ManuallyPromote` which allows you to choose when to promote the new generation. This means that you can minimize the impact of potential downtime. +- **Manual Promotion** (*Materialize Self-Managed only*): [Rollout strategies](/self-managed-deployments/upgrading/#rollout-strategies) allow you control how Materialize transitions from the current generation to a new generation during an upgrade. We've added a new rollout strategy called `ManuallyPromote` which allows you to choose when to promote the new generation. This means that you can minimize the impact of potential downtime. ### Bug Fixes {#v26.4-bug-fixes} - Fixed timestamp determination logic to handle empty read holds correctly. @@ -1373,7 +1373,7 @@ use the new setting `rolloutStrategy` to specify either: - `WaitUntilReady` (*Default*) - `ImmediatelyPromoteCausingDowntime` -For more information, see [`rolloutStrategy`](/self-managed-deployments/upgrading/materialize-instances/v1/#rollout-strategies). +For more information, see [`rolloutStrategy`](/self-managed-deployments/upgrading/#rollout-strategies). ### Terraform helpers diff --git a/doc/user/content/security/self-managed/authentication.md b/doc/user/content/security/self-managed/authentication.md index 0d1361c109995..ed831ce4310cf 100644 --- a/doc/user/content/security/self-managed/authentication.md +++ b/doc/user/content/security/self-managed/authentication.md @@ -51,7 +51,9 @@ The following example Kubernetes manifest includes configuration for SASL/SCRAM-SHA-256 authentication: {{< tabs >}} -{{< tab "v1 (v26.30+)" >}} +{{< tab "v1alpha1" >}} + +{{< self-managed/crd-version-note "v1alpha1" >}} ```hc {hl_lines="15 25"} apiVersion: v1 @@ -70,19 +72,22 @@ stringData: license_key: "..." external_login_password_mz_system: "enter_mz_system_password" --- -apiVersion: materialize.cloud/v1 +apiVersion: materialize.cloud/v1alpha1 kind: Materialize metadata: name: 12345678-1234-1234-1234-123456789012 namespace: materialize-environment spec: - environmentdImageRef: materialize/environmentd:v26.12.1 + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} backendSecretName: materialize-backend authenticatorKind: Sasl + requestRollout: 00000000-0000-0000-0000-000000000003 # Enabling auth on an existing instance requires a rollout ``` {{< /tab >}} -{{< tab "v1alpha1 (before v26.30)" >}} +{{< tab "v1" >}} + +{{< self-managed/crd-version-note "v1" >}} ```hc {hl_lines="15 25"} apiVersion: v1 @@ -101,13 +106,13 @@ stringData: license_key: "..." external_login_password_mz_system: "enter_mz_system_password" --- -apiVersion: materialize.cloud/v1alpha1 +apiVersion: materialize.cloud/v1 kind: Materialize metadata: name: 12345678-1234-1234-1234-123456789012 namespace: materialize-environment spec: - environmentdImageRef: materialize/environmentd:v26.12.1 + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} backendSecretName: materialize-backend authenticatorKind: Sasl ``` @@ -135,7 +140,9 @@ The following example Kubernetes manifest includes configuration for password authentication: {{< tabs >}} -{{< tab "v1 (v26.30+)" >}} +{{< tab "v1alpha1" >}} + +{{< self-managed/crd-version-note "v1alpha1" >}} ```hc {hl_lines="15 25"} apiVersion: v1 @@ -154,19 +161,22 @@ stringData: license_key: "..." external_login_password_mz_system: "enter_mz_system_password" --- -apiVersion: materialize.cloud/v1 +apiVersion: materialize.cloud/v1alpha1 kind: Materialize metadata: name: 12345678-1234-1234-1234-123456789012 namespace: materialize-environment spec: - environmentdImageRef: materialize/environmentd:v26.12.1 + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} backendSecretName: materialize-backend authenticatorKind: Password + requestRollout: 00000000-0000-0000-0000-000000000003 # Enabling auth on an existing instance requires a rollout ``` {{< /tab >}} -{{< tab "v1alpha1 (before v26.30)" >}} +{{< tab "v1" >}} + +{{< self-managed/crd-version-note "v1" >}} ```hc {hl_lines="15 25"} apiVersion: v1 @@ -185,13 +195,13 @@ stringData: license_key: "..." external_login_password_mz_system: "enter_mz_system_password" --- -apiVersion: materialize.cloud/v1alpha1 +apiVersion: materialize.cloud/v1 kind: Materialize metadata: name: 12345678-1234-1234-1234-123456789012 namespace: materialize-environment spec: - environmentdImageRef: materialize/environmentd:v26.12.1 + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} backendSecretName: materialize-backend authenticatorKind: Password ``` diff --git a/doc/user/content/security/self-managed/sso.md b/doc/user/content/security/self-managed/sso.md index d1bb8cae88fed..77dca56b76cf3 100644 --- a/doc/user/content/security/self-managed/sso.md +++ b/doc/user/content/security/self-managed/sso.md @@ -212,6 +212,11 @@ following fields: The following example Kubernetes manifest includes configuration for OIDC authentication: +{{< tabs >}} +{{< tab "v1alpha1" >}} + +{{< self-managed/crd-version-note "v1alpha1" >}} + ```yaml {hl_lines="6-15 26 36-38"} apiVersion: v1 kind: Namespace @@ -246,7 +251,7 @@ metadata: name: 12345678-1234-1234-1234-123456789012 namespace: materialize-environment spec: - environmentdImageRef: materialize/environmentd:v26.26.0 # Use v26.26.0 or later + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} backendSecretName: materialize-backend authenticatorKind: Oidc requestRollout: 00000000-0000-0000-0000-000000000003 # Switching to Oidc requires a rollout @@ -254,9 +259,63 @@ spec: ``` Apply the updated manifest to your Kubernetes cluster. See -[Upgrading](/self-managed-deployments/upgrading/materialize-instances/v1alpha1/#rollout-configuration) for +[Upgrading](/self-managed-deployments/upgrading/#rollout-configuration) for details on rollout configuration. +{{< /tab >}} +{{< tab "v1" >}} + +{{< self-managed/crd-version-note "v1" >}} + +```yaml {hl_lines="6-15 26 36-37"} +apiVersion: v1 +kind: Namespace +metadata: + name: materialize-environment +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mz-system-params + namespace: materialize-environment +data: + # Create an empty system parameter configmap for later steps + system-params.json: | + { + } +--- +apiVersion: v1 +kind: Secret +metadata: + name: materialize-backend + namespace: materialize-environment +stringData: + metadata_backend_url: "..." + persist_backend_url: "..." + license_key: "..." + external_login_password_mz_system: "enter_mz_system_password" +--- +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} + backendSecretName: materialize-backend + authenticatorKind: Oidc + systemParameterConfigmapName: mz-system-params +``` + +Apply the updated manifest to your Kubernetes cluster. With the `v1` CRD, +rollouts trigger automatically when spec fields change, so no `requestRollout` +is needed. See +[Upgrading](/self-managed-deployments/upgrading/#rollout-configuration) +for details on rollout configuration. + +{{< /tab >}} +{{< /tabs >}} + {{% include-headless "/headless/self-managed-deployments/enabled-auth-setting-warning" %}} diff --git a/doc/user/content/self-managed-deployments/_index.md b/doc/user/content/self-managed-deployments/_index.md index 0980528be7ee4..03d46e946cc50 100644 --- a/doc/user/content/self-managed-deployments/_index.md +++ b/doc/user/content/self-managed-deployments/_index.md @@ -158,10 +158,12 @@ Materialize CR, see [Materialize CRD Field Descriptions](/self-managed-deployments/materialize-crd-field-descriptions/). {{< tabs >}} -{{< tab "v1 (v26.30+)" >}} +{{< tab "v1alpha1" >}} + +{{< self-managed/crd-version-note "v1alpha1" >}} ```yaml -apiVersion: materialize.cloud/v1 +apiVersion: materialize.cloud/v1alpha1 kind: Materialize metadata: name: 12345678-1234-1234-1234-123456789012 @@ -172,10 +174,12 @@ spec: ``` {{< /tab >}} -{{< tab "v1alpha1 (before v26.30)" >}} +{{< tab "v1" >}} + +{{< self-managed/crd-version-note "v1" >}} ```yaml -apiVersion: materialize.cloud/v1alpha1 +apiVersion: materialize.cloud/v1 kind: Materialize metadata: name: 12345678-1234-1234-1234-123456789012 @@ -194,13 +198,9 @@ creates all required Kubernetes resources. #### Modifying the custom resource {{< tabs >}} -{{< tab "v1 (v26.30+)" >}} - -To modify a custom resource, update the CRD with your changes. -When you apply the CRD, the operator will roll out the changes. +{{< tab "v1alpha1" >}} -{{< /tab >}} -{{< tab "v1alpha1 (before v26.30)" >}} +{{< self-managed/crd-version-note "v1alpha1" >}} To modify a custom resource, update the CRD with your changes, including the `requestRollout` field with a new UUID value. When you apply the CRD, the @@ -211,6 +211,14 @@ If you do not specify a new `requestRollout` UUID, the operator watches for updates but does not roll out the changes. {{< /note >}} +{{< /tab >}} +{{< tab "v1" >}} + +{{< self-managed/crd-version-note "v1" >}} + +To modify a custom resource, update the CRD with your changes. +When you apply the CRD, the operator will roll out the changes. + {{< /tab >}} {{< /tabs >}} diff --git a/doc/user/content/self-managed-deployments/configuration-system-parameters.md b/doc/user/content/self-managed-deployments/configuration-system-parameters.md index 44aa90b7bbe23..1c2e5969153c7 100644 --- a/doc/user/content/self-managed-deployments/configuration-system-parameters.md +++ b/doc/user/content/self-managed-deployments/configuration-system-parameters.md @@ -68,34 +68,38 @@ Reference the ConfigMap in your Materialize custom resource by setting the `systemParameterConfigmapName` field to the name of your ConfigMap: {{< tabs >}} -{{< tab "v1 (v26.30+)" >}} +{{< tab "v1alpha1" >}} -```yaml {hl_lines="9"} -apiVersion: materialize.cloud/v1 +{{< self-managed/crd-version-note "v1alpha1" >}} + +```yaml {hl_lines="9-10"} +apiVersion: materialize.cloud/v1alpha1 kind: Materialize metadata: name: 12345678-1234-1234-1234-123456789012 namespace: materialize-environment spec: - environmentdImageRef: materialize/environmentd:v26.0.0 + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} backendSecretName: materialize-backend systemParameterConfigmapName: mz-system-params + requestRollout: 00000000-0000-0000-0000-000000000003 # Changing the CR requires a rollout ``` {{< /tab >}} -{{< tab "v1alpha1 (before v26.30)" >}} +{{< tab "v1" >}} -```yaml {hl_lines="9-10"} -apiVersion: materialize.cloud/v1alpha1 +{{< self-managed/crd-version-note "v1" >}} + +```yaml {hl_lines="9"} +apiVersion: materialize.cloud/v1 kind: Materialize metadata: name: 12345678-1234-1234-1234-123456789012 namespace: materialize-environment spec: - environmentdImageRef: materialize/environmentd:v26.0.0 + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} backendSecretName: materialize-backend systemParameterConfigmapName: mz-system-params - requestRollout: 00000000-0000-0000-0000-000000000003 # Changing the CR requires a rollout ``` {{< /tab >}} @@ -151,10 +155,12 @@ Materialize custom resource YAML and update it whenever you need to force a ConfigMap reload: {{< tabs >}} -{{< tab "v1 (v26.30+)" >}} +{{< tab "v1alpha1" >}} + +{{< self-managed/crd-version-note "v1alpha1" >}} ```yaml -apiVersion: materialize.cloud/v1 +apiVersion: materialize.cloud/v1alpha1 kind: Materialize metadata: name: 12345678-1234-1234-1234-123456789012 @@ -166,10 +172,12 @@ spec: ``` {{< /tab >}} -{{< tab "v1alpha1 (before v26.30)" >}} +{{< tab "v1" >}} + +{{< self-managed/crd-version-note "v1" >}} ```yaml -apiVersion: materialize.cloud/v1alpha1 +apiVersion: materialize.cloud/v1 kind: Materialize metadata: name: 12345678-1234-1234-1234-123456789012 diff --git a/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md b/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md index 5c30e1cd52c8e..1bce6a438ebbf 100644 --- a/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md +++ b/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md @@ -107,7 +107,7 @@ Starting in v26.0, Self-Managed Materialize requires a license key. kubectl get nodes --show-labels ``` -1. Recommended: Install cert-manager +1. Install cert-manager Cert-manager is used for generating TLS certificates needed by the materialize operator for CRD conversion webhooks. It is currently only required if you enable the v1 @@ -287,6 +287,14 @@ Starting in v26.0, Self-Managed Materialize requires a license key. kubectl apply -f sample-materialize.yaml ``` + {{< tip >}} + **Optional — use the `v1` CRD.** If you enabled the `v1` CRD on the + operator, edit `sample-materialize.yaml` to set `apiVersion: + materialize.cloud/v1` and remove any `requestRollout` field before + applying. See [Adopting the v1 + CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). + {{< /tip >}} + 1. Verify the installation and check the status: {{< note >}} diff --git a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/_index.md b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/_index.md index 35ada81f0c88d..ddd5303da8d57 100644 --- a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/_index.md +++ b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/_index.md @@ -13,5 +13,5 @@ aliases: Select the CRD API version for your Materialize deployment: -- [v1 (v26.30+)](/self-managed-deployments/materialize-crd-field-descriptions/v1/) +- [v1 (available starting v26.30+)](/self-managed-deployments/materialize-crd-field-descriptions/v1/) - [v1alpha1 (before v26.30)](/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1/) diff --git a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md index 403155796ec5c..5141a555fd1c2 100644 --- a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md +++ b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md @@ -7,4 +7,11 @@ menu: weight: 10 --- +{{< note >}} +The `v1` CRD is available starting in v26.30 and is opt-in. To get the new +automatic-rollout behavior for an existing instance, see [Adopting the v1 +CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). +For the earlier API, see [v1alpha1](/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1/). +{{}} + {{% self-managed/materialize-crd-descriptions-v1 %}} diff --git a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md index 41959aaa02545..3ebe6dd703642 100644 --- a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md +++ b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md @@ -7,4 +7,11 @@ menu: weight: 20 --- +{{< note >}} +`v1alpha1` is the default CRD version (used before v26.30 and still the default). +Starting in v26.30, the [v1](/self-managed-deployments/materialize-crd-field-descriptions/v1/) +CRD is available; to get its automatic-rollout behavior, see [Adopting the v1 +CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). +{{}} + {{% self-managed/materialize-crd-descriptions-v1alpha1 %}} diff --git a/doc/user/content/self-managed-deployments/upgrading/_index.md b/doc/user/content/self-managed-deployments/upgrading/_index.md index 77b5326918b40..de267f6dee9b7 100644 --- a/doc/user/content/self-managed-deployments/upgrading/_index.md +++ b/doc/user/content/self-managed-deployments/upgrading/_index.md @@ -93,10 +93,288 @@ helm upgrade -n materialize my-demo materialize/operator \ ## Upgrading Materialize Instances -Select the instructions for your CRD API version: +After upgrading the operator, upgrade each Materialize instance to the operator's +**App Version** by updating its `environmentdImageRef`. For step-by-step +instructions, use the [upgrade guide](#upgrade-guides) for your environment +(Kind, AWS, Azure, or GCP). This section describes the rollout behavior those +guides rely on. -- [v1 (v26.30+)](/self-managed-deployments/upgrading/materialize-instances/v1/) -- [v1alpha1 (before v26.30)](/self-managed-deployments/upgrading/materialize-instances/v1alpha1/) +### CRD API versions + +Starting in v26.30, the Materialize Operator supports two CRD API versions: + +- **v1alpha1** (default) uses a two-step rollout: first stage the spec change, + then trigger a rollout with a new `requestRollout` UUID. +- **v1** simplifies upgrades — rollouts trigger automatically when spec fields + change, removing the need to manually set a `requestRollout` UUID. + +Adopting v1 is **opt-in** for now. Upgrading the operator to v26.30+ does not +change your existing v1alpha1 CRs or their behavior; you can continue using +v1alpha1 and adopt v1 for individual instances at your own pace. + +{{< important >}} +In the next major release, all Materialize CRs will be force upgraded to v1. You +will still be able to apply v1alpha1 CRs, but they will be auto-converted to v1 +and use the v1 rollout behavior. We recommend opting in to v1 at your convenience +to migrate on your own schedule before the upgrade is mandatory. +{{< /important >}} + +## Adopting the v1 CRD + +Adopting v1 is opt-in and does not trigger a rollout on its own. Before you +begin, ensure you have completed the prerequisites in the [v26.30 upgrade +notes](#upgrading-to-v2630-and-later-versions) (cert-manager, network/firewall +changes), and have enabled the v1 CRD by setting the Helm value +`operator.args.installV1CRD=true` on the operator. Without this value, the +operator only installs the v1alpha1 CRD version, and the Kubernetes API server +rejects v1 CRs. + +### How it works + +The v1alpha1 CRD remains the storage version. When you submit a v1 CR, the +operator's conversion webhook automatically converts it to v1alpha1 for storage. +During conversion, the webhook computes a SHA256 hash of a subset of the spec +fields and derives a deterministic `requestRollout` UUID from it. The hash covers +the fields that affect the running `environmentd` (for example, +`environmentdImageRef`, `environmentdExtraArgs`, `environmentdExtraEnv`, resource +requirements, `podAnnotations`, `podLabels`, `authenticatorKind`, `enableRbac`, +`rolloutStrategy`, and `forceRollout`). It excludes fields that do not require a +rollout, such as `balancerd`/`console` resource requirements and replica counts. +The operator triggers a rollout whenever `requestRollout` differs from the last +completed rollout, so: + +- **Adopting v1 on an existing v1alpha1 instance typically triggers one + rollout.** The derived `requestRollout` is computed from the spec hash and will + not match the `requestRollout` you previously set by hand, so the instance + rolls out once even if nothing else in the spec changed. Adopt v1 during a + window where a rollout is acceptable. +- **Once on v1, an unchanged spec does not trigger a rollout.** Reapplying the + same spec produces the same hash and the same derived `requestRollout`. + Changing a hashed spec field produces a new value and triggers a rollout + automatically. + +### Using kubectl + +To adopt v1 for an existing instance, apply your CR with `apiVersion: +materialize.cloud/v1` and remove the `requestRollout` field: + +```shell +kubectl apply -f - < + namespace: +spec: + environmentdImageRef: + backendSecretName: + # ... other spec fields (copy from your existing CR, removing requestRollout) +EOF +``` + +### Using Terraform + +If you are managing your Materialize instance with the [Materialize Terraform +modules](https://github.com/MaterializeInc/materialize-terraform-self-managed), +set: + +```hcl +crd_version = "v1" +request_rollout = null +``` + +### Returning to the v1alpha1 behavior + +You can go back to the v1alpha1 rollout behavior at any time by applying your CR +with `apiVersion: materialize.cloud/v1alpha1` and an explicit `requestRollout` +UUID. + +## Rollout configuration + +How you trigger a rollout depends on your CRD API version. + +{{< tabs >}} +{{< tab "v1alpha1" >}} + +Specify a new `UUID` value for `requestRollout` to roll out changes to the +Materialize instance. + +{{< note >}} +`requestRollout` without the `forceRollout` field only rolls out if changes exist +to the Materialize instance. To roll out even if there are no changes to the +instance, use it with `forceRollout`. +{{< /note >}} + +```shell +# Only rolls out if there are changes +kubectl patch materialize \ + -n \ + --type='merge' \ + -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\"}}" +``` + +To force a rollout even when there are no other changes, set a new `forceRollout` +UUID alongside `requestRollout`: + +```shell +kubectl patch materialize \ + -n \ + --type='merge' \ + -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\", \"forceRollout\": \"$(uuidgen)\"}}" +``` + +{{< /tab >}} +{{< tab "v1" >}} + +With v1, the operator computes a hash of the spec fields and automatically +triggers a rollout when the hash changes — there is no `requestRollout` field. + +Specify a new `UUID` value for `forceRollout` to trigger a rollout even when +there are no other changes to the instance. The `forceRollout` value is included +in the hash, so changing it produces a new hash and triggers a rollout. Set it in +your manifest and reapply: + +```yaml +spec: + forceRollout: # e.g. the output of `uuidgen` +``` + +```shell +kubectl apply -f materialize.yaml +``` + +{{< /tab >}} +{{< /tabs >}} + +## Rollout strategies + +Rollout strategies control how Materialize transitions from the current +generation to a new generation during an upgrade. The behavior follows your +`rolloutStrategy` setting. + +### *WaitUntilReady* — ***Default*** + +`WaitUntilReady` creates a new generation of pods and automatically promotes them +as soon as they catch up to the old generation and become `ReadyToPromote`. +Because both generations run simultaneously until the promotion, this strategy +temporarily doubles the required resources to run Materialize. + +{{< note >}} +While the new generation waits to become `ReadyToPromote`, it runs in a +read-only, un-promoted state and holds back compaction. To prevent it from +sitting in this state indefinitely (which can cause incident-inducing load when +it is eventually promoted), the rollout is bounded by the `rolloutRequestTimeout` +field in the Materialize spec, which defaults to `24h`. + +If the new generation does not become `ReadyToPromote` within +`rolloutRequestTimeout`, the operator cancels the rollout: the new generation is +torn down and the previously-active generation continues serving. You can then +trigger a new rollout (in v1, set a new `forceRollout`; in v1alpha1, set a new +`requestRollout`). +{{< /note >}} + +### *ImmediatelyPromoteCausingDowntime* + +{{< warning >}} Using the `ImmediatelyPromoteCausingDowntime` rollout flag will cause downtime. {{< /warning >}} + +`ImmediatelyPromoteCausingDowntime` tears down the prior generation and +immediately promotes the new generation without waiting for it to hydrate. This +causes downtime until the new generation has hydrated. However, it does not +require additional resources. + +### *ManuallyPromote* + +`ManuallyPromote` allows you to choose when to promote the new generation. This +means you can time the promotion for periods when load is low, minimizing the +impact of potential downtime for any clients connected to Materialize. This +strategy temporarily doubles the required resources to run Materialize. + +To minimize downtime, wait until the new generation has fully hydrated and caught +up to the prior generation before promoting. To check hydration status, inspect +the `UpToDate` condition in the Materialize resource status. When hydration +completes, the condition will be `ReadyToPromote`. + +To promote, update the `forcePromote` field to match the current rollout +identifier (in v1, the `status.requestedRolloutHash`; in v1alpha1, the +`requestRollout` UUID in the spec). If you need to promote before hydration +completes, you can set `forcePromote` immediately, but clients may experience +downtime. + +{{< warning >}} Leaving a new generation unpromoted for over 6 hours may cause downtime. {{< /warning >}} + +**Do not leave new generations unpromoted indefinitely**. They should either be +promoted or canceled. New generations open a read hold on the metadata database +that prevents compaction. This hold is only released when the generation is +promoted or canceled. If left open too long, promoting or canceling can trigger a +spike in deletion load on the metadata database, potentially causing downtime. It +is not recommended to leave generations unpromoted for over 6 hours. + +### *inPlaceRollout* — ***Deprecated*** (v1alpha1 only) + +The setting is ignored. + +## Verifying the upgrade + +After initiating the rollout, you can monitor the status field of the Materialize +custom resource to check on the upgrade. + +```shell +# Watch the status of your Materialize environment +kubectl get materialize -n materialize-environment -w + +# Check the logs of the operator +kubectl logs -l app.kubernetes.io/name=materialize-operator -n materialize +``` + +## Cancelling the upgrade + +You may want to cancel an in-progress rollout if the upgrade has failed (for +example, new pods are not healthy). Before cancelling, verify that the upgrade has +not already completed by checking that the deploy generation (found via +`status.activeGeneration`) is still the one from before the upgrade. Once an +upgrade has already happened, you cannot revert using this method. + +{{< tabs >}} +{{< tab "v1alpha1" >}} + +To cancel an in-progress rollout and revert to the last completed rollout state, +revert both `requestRollout` and `environmentdImageRef` back to the values from +the last completed rollout. Reverting `environmentdImageRef` alongside +`requestRollout` keeps the spec aligned with what is actually running, so a later +rollout doesn't accidentally pick up the previously attempted upgrade image. + +First, retrieve the last completed rollout request ID and the matching +environmentd image ref from your Materialize CR: + +```shell +kubectl get materialize -n materialize-environment \ + -o jsonpath='{.status.lastCompletedRolloutRequest} {.status.lastCompletedRolloutEnvironmentdImageRef}' +``` + +Then set both fields back to these values in a single patch: + +```shell +kubectl patch materialize \ + -n materialize-environment \ + --type='merge' \ + -p "{\"spec\": {\"requestRollout\": \"\", \"environmentdImageRef\": \"\"}}" +``` + +{{< /tab >}} +{{< tab "v1" >}} + +To cancel an in-progress rollout and revert to the last completed rollout state, +reapply the Materialize resource with the spec it had before the rollout (notably +the previous `environmentdImageRef`). Because v1 derives the rollout from the +spec hash, restoring the previous spec produces the previous hash and returns the +instance to the last completed state. + +```shell +kubectl apply -f previous_materialize_configuration.yaml +``` + +{{< /tab >}} +{{< /tabs >}} ## Version Specific Upgrade Notes diff --git a/doc/user/content/self-managed-deployments/upgrading/materialize-instances/_index.md b/doc/user/content/self-managed-deployments/upgrading/materialize-instances/_index.md deleted file mode 100644 index 6fccc4ac55a86..0000000000000 --- a/doc/user/content/self-managed-deployments/upgrading/materialize-instances/_index.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -title: "Upgrading Materialize Instances" -description: "Upgrading Materialize instances for Self-Managed deployments." -disable_list: true -menu: - main: - parent: "upgrading" - identifier: "upgrading-materialize-instances" - weight: 20 ---- - -{{< important >}} - -When upgrading Materialize, always upgrade the Helm Chart and Materialize -Operator first. See [Upgrading the Helm Chart and Materialize Operator](/self-managed-deployments/upgrading/#upgrading-the-helm-chart-and-materialize-operator). - -{{}} - -## CRD API Versions - -Starting in v26.30, the Materialize Operator supports two CRD API versions: - -- **v1** simplifies the upgrade process. Rollouts trigger automatically when spec fields change, removing the need to manually set a `requestRollout` UUID. -- **v1alpha1** uses the original two-step upgrade process: first stage changes, then trigger a rollout with a new `requestRollout` UUID. - -Switching to v1 is **opt-in** for now. Upgrading the operator to v26.30+ does not change your existing v1alpha1 CRs or their behavior. You can continue using v1alpha1 until the next major release, switching individual instances to v1 at your own pace. - -{{< important >}} -In the next major release, all Materialize CRs will be force upgraded to v1. You will still be able to apply v1alpha1 CRs, but they will be auto-converted to v1 and use the v1 rollout behavior. We recommend opting in to v1 at your convenience to migrate on your own schedule before the upgrade is mandatory. -{{}} - -Select the instructions for your CRD API version: - -- [v1 (v26.30+)](/self-managed-deployments/upgrading/materialize-instances/v1/) -- [v1alpha1 (before v26.30)](/self-managed-deployments/upgrading/materialize-instances/v1alpha1/) - -## Switching from v1alpha1 to v1 - -Switching to v1 is opt-in and does not trigger a rollout on its own. Before switching, ensure you have completed the prerequisites in the [v26.30 upgrade notes](/self-managed-deployments/upgrading/#upgrading-to-v2630-and-later-versions) (cert-manager, network/firewall changes), and have enabled the v1 CRD by setting the Helm value `operator.args.installV1CRD=true` on the operator. Without this value, the operator only installs the v1alpha1 CRD version, and the Kubernetes API server rejects v1 CRs. - -### How it works - -The v1alpha1 CRD remains the storage version. When you submit a v1 CR, the operator's conversion webhook automatically converts it to v1alpha1 for storage. During conversion, the webhook computes a SHA256 hash of the spec and derives a deterministic `requestRollout` UUID from it. This means: - -- If the spec hasn't changed, the same UUID is generated, so **no unintended rollout is triggered** by switching API versions alone. -- If the spec has changed, a different UUID is produced, automatically triggering a rollout. - -### Using kubectl - -To switch an existing instance to v1, apply your CR with the updated `apiVersion` and remove the `requestRollout` field: - -```shell -kubectl apply -f - < - namespace: -spec: - environmentdImageRef: - backendSecretName: - # ... other spec fields (copy from your existing CR, removing requestRollout) -EOF -``` - -Or patch the API version on an existing CR: - -```shell -kubectl patch materialize \ - -n \ - --type='merge' \ - -p '{"apiVersion":"materialize.cloud/v1"}' -``` - -### Using Terraform - -If you are managing your Materialize instance with the [Materialize Terraform modules](https://github.com/MaterializeInc/materialize-terraform-self-managed), set: - -```hcl -crd_version = "v1" -request_rollout = null -``` - -### Switching back to v1alpha1 - -You can switch back to v1alpha1 at any time by reapplying your CR with `apiVersion: materialize.cloud/v1alpha1` and an explicit `requestRollout` UUID. diff --git a/doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1.md b/doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1.md deleted file mode 100644 index a8cd458f1352f..0000000000000 --- a/doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1.md +++ /dev/null @@ -1,173 +0,0 @@ ---- -title: "v1" -description: "Upgrading Self-Managed Materialize instances using the v1 CRD API (v26.30+)." -menu: - main: - parent: "upgrading-materialize-instances" - identifier: "upgrading-v1" - weight: 10 ---- - -{{< note >}} -If you are currently using v1alpha1 and want to switch, see [Switching from v1alpha1 to v1](/self-managed-deployments/upgrading/materialize-instances/#switching-from-v1alpha1-to-v1). -{{}} - -## Upgrading Materialize Instances - -With the `v1` CRD, rollouts trigger automatically when spec fields change. -Unlike `v1alpha1`, there is no `requestRollout` field. You only need to update the spec and the operator handles the rest. - -**After** you have upgraded your Materialize Operator, upgrade your Materialize -instance(s) to the **APP Version** of the Operator. To find the version of your -currently deployed Materialize Operator: - -```shell -helm list -n materialize -``` - -You will use the returned **App Version** for the updated `environmentdImageRef` -value. Specifically, for your Materialize instance(s), set -`environmentdImageRef` value to use the new version: - -``` -spec: - environmentdImageRef: docker.io/materialize/environmentd: -``` - -#### Using `kubectl patch` - -To initiate the Materialize instance version upgrade, you can -use the `kubectl patch` command; for example, if the **App Version** is {{< self-managed/versions/get-latest-version >}}: - -```shell -kubectl patch materialize \ - -n \ - --type='merge' \ - -p "{\"apiVersion\": \"materialize.cloud/v1\", \"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\"}}" -``` - - -#### Using YAML Definition - -Alternatively, you can update your Materialize custom resource definition directly: - -```yaml -apiVersion: materialize.cloud/v1 -kind: Materialize -metadata: - name: 12345678-1234-1234-1234-123456789012 - namespace: materialize-environment -spec: - environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} # Update version as needed - forceRollout: 33333333-3333-3333-3333-333333333333 # Optional: for forced rollouts - rolloutStrategy: WaitUntilReady # The mechanism to use when rolling out the new version. Can be WaitUntilReady or ImmediatelyPromoteCausingDowntime - backendSecretName: materialize-backend -``` - -Apply the updated definition: - -```shell -kubectl apply -f materialize.yaml -``` - -#### Using Terraform - -If you are managing your Materialize instance with the -[Materialize Terraform modules](https://github.com/MaterializeInc/materialize-terraform-self-managed), -update the `environmentd_version` variable: - -```hcl -module "materialize_instance" { - source = "../kubernetes/modules/materialize-instance" - - crd_version = "v1" - request_rollout = null - environmentd_version = "{{< self-managed/versions/get-latest-version >}}" # Update to the new version - # ... other configuration -} -``` - -## Rollout Configuration - -With `v1`, the operator computes a hash of the spec fields and automatically triggers a rollout when the hash changes. The `requestRollout` field from `v1alpha1` is not used. - -### `forceRollout` - -Specify a new `UUID` value for `forceRollout` to trigger a rollout even when there are -no other changes to the instance. The `forceRollout` value is included in the -hash calculation, so changing it produces a new hash and triggers a rollout. - -```shell -kubectl patch materialize \ - -n materialize-environment \ - --type='merge' \ - -p "{\"apiVersion\": \"materialize.cloud/v1\", \"spec\": {\"forceRollout\": \"$(uuidgen)\"}}" -``` - -### Rollout strategies - -Rollout strategies control how Materialize transitions from the current generation to a new generation during an upgrade. - -The behavior of the new version rollout follows your `rolloutStrategy` setting. - -#### *WaitUntilReady* - ***Default*** - -`WaitUntilReady` creates a new generation of pods and automatically promotes -them as soon as they catch up to the old generation and become `ReadyToPromote`. -Because both generations run simultaneously until the promotion, this strategy -temporarily doubles the required resources to run Materialize. - -{{< note >}} - -While the new generation waits to become `ReadyToPromote`, it runs in a -read-only, un-promoted state and holds back compaction. To prevent it from -sitting in this state indefinitely (which can cause incident-inducing load when -it is eventually promoted), the rollout is bounded by the `rolloutRequestTimeout` -field in the Materialize spec, which defaults to `24h`. - -If the new generation does not become `ReadyToPromote` within -`rolloutRequestTimeout`, the operator cancels the rollout: the new generation is -torn down and the previously-active generation continues serving. You can then -trigger a new rollout by setting `forceRollout` to a new value. - -{{< /note >}} - -#### *ImmediatelyPromoteCausingDowntime* -{{< warning >}} Using the `ImmediatelyPromoteCausingDowntime` rollout flag will cause downtime. {{< /warning >}} - -`ImmediatelyPromoteCausingDowntime` tears down the prior generation, and immediately promotes the new generation without waiting for it to hydrate. This causes downtime until the new generation has hydrated. However, it does not require additional resources. - -#### *ManuallyPromote* - -`ManuallyPromote` allows you to choose when to promote the new generation. This means you can time the promotion for periods when load is low, minimizing the impact of potential downtime for any clients connected to Materialize. This strategy temporarily doubles the required resources to run Materialize. - -To minimize downtime, wait until the new generation has fully hydrated and caught up to the prior generation before promoting. To check hydration status, inspect the `UpToDate` condition in the Materialize resource status. When hydration completes, the condition will be `ReadyToPromote`. - -To promote, update the `forcePromote` field to match the `requestedRolloutHash` field in the Materialize status. If you need to promote before hydration completes, you can set `forcePromote` immediately, but clients may experience downtime. - -{{< warning >}} Leaving a new generation unpromoted for over 6 hours may cause downtime. {{< /warning >}} - -**Do not leave new generations unpromoted indefinitely**. They should either be promoted or canceled. New generations open a read hold on the metadata database that prevents compaction. This hold is only released when the generation is promoted or canceled. If left open too long, promoting or canceling can trigger a spike in deletion load on the metadata database, potentially causing downtime. It is not recommended to leave generations unpromoted for over 6 hours. - - -## Verifying the Upgrade - -After initiating the rollout, you can monitor the status field of the Materialize custom resource to check on the upgrade. - -```shell -# Watch the status of your Materialize environment -kubectl get materialize -n materialize-environment -w - -# Check the logs of the operator -kubectl logs -l app.kubernetes.io/name=materialize-operator -n materialize -``` - -## Cancelling the Upgrade - -You may want to cancel an in-progress rollout if the upgrade has failed. This may be indicated by new pods not being healthy. Before cancelling, verify that the upgrade has not already completed by checking that the deploy generation (found via `status.activeGeneration`) is still the one from before the upgrade. Once an upgrade has already happened, you cannot revert using this method. - -To cancel an in-progress rollout and revert to the last completed rollout state, you must revert the Materialize resource to the contents it had before triggering the rollout: - -```shell -kubectl apply -f previous_materialize_configuration.yaml -``` diff --git a/doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1alpha1.md b/doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1alpha1.md deleted file mode 100644 index 15b9b72e22cc5..0000000000000 --- a/doc/user/content/self-managed-deployments/upgrading/materialize-instances/v1alpha1.md +++ /dev/null @@ -1,234 +0,0 @@ ---- -title: "v1alpha1" -description: "Upgrading Self-Managed Materialize instances using the v1alpha1 CRD API (before v26.30)." -menu: - main: - parent: "upgrading-materialize-instances" - identifier: "upgrading-v1alpha1" - weight: 20 -aliases: - - /self-managed-deployments/upgrading/v1alpha1-upgrade-instructions/ ---- - -## Upgrading Materialize Instances - -**After** you have upgraded your Materialize Operator, upgrade your Materialize -instance(s) to the **APP Version** of the Operator. To find the version of your -currently deployed Materialize Operator: - -```shell -helm list -n materialize -``` - -You will use the returned **App Version** for the updated `environmentdImageRef` -value. Specifically, for your Materialize instance(s), set -`environmentdImageRef` value to use the new version: - -``` -spec: - environmentdImageRef: docker.io/materialize/environmentd: -``` - -To minimize unexpected downtime and avoid connection drops at critical -periods for your application, the upgrade process involves two steps: - -- First, stage the changes (update the `environmentdImageRef` with the new - version) to the Materialize custom resource. The Operator watches for changes - but does not automatically roll out the changes. - -- Second, roll out the changes by specifying a new UUID for `requestRollout`. - -### Stage the Materialize instance version change - -To stage the Materialize instances version upgrade, update the -`environmentdImageRef` field in the Materialize custom resource spec to the -compatible version of your currently deployed Materialize Operator. - -To stage, but **not** rollout, the Materialize instance version upgrade, you can -use the `kubectl patch` command; for example, if the **App Version** is {{< self-managed/versions/get-latest-version >}}: - -```shell -kubectl patch materialize \ - -n \ - --type='merge' \ - -p "{\"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\"}}" -``` - -{{< note >}} -Until you specify a new `requestRollout`, the Operator watches for updates but -does not roll out the changes. -{{< /note >}} - - -### Applying the changes via `requestRollout` - -To apply the Materialize instance upgrade, you must update the `requestRollout` field in the Materialize custom resource spec to a new UUID. -Be sure to consult the [Rollout Configurations](#rollout-configuration) to ensure you've selected the correct rollout behavior. -```shell -# Then trigger the rollout with a new UUID -kubectl patch materialize \ - -n \ - --type='merge' \ - -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\"}}" -``` - -### Staging and applying in a single command - -Although separating the staging and rollout of the changes into two steps can -minimize unexpected downtime and avoid connection drops at critical periods, you -can, if preferred, combine both operations in a single command - -```shell -kubectl patch materialize \ - -n materialize-environment \ - --type='merge' \ - -p "{\"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\", \"requestRollout\": \"$(uuidgen)\"}}" -``` - -#### Using YAML Definition - -Alternatively, you can update your Materialize custom resource definition directly: - -```yaml -apiVersion: materialize.cloud/v1alpha1 -kind: Materialize -metadata: - name: 12345678-1234-1234-1234-123456789012 - namespace: materialize-environment -spec: - environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} # Update version as needed - requestRollout: 22222222-2222-2222-2222-222222222222 # Use a new UUID - forceRollout: 33333333-3333-3333-3333-333333333333 # Optional: for forced rollouts - inPlaceRollout: false # In Place rollout is deprecated and ignored. Please use rolloutStrategy - rolloutStrategy: WaitUntilReady # The mechanism to use when rolling out the new version. Can be WaitUntilReady or ImmediatelyPromoteCausingDowntime - backendSecretName: materialize-backend -``` - -Apply the updated definition: - -```shell -kubectl apply -f materialize.yaml -``` - -## Rollout Configuration - -### `requestRollout` - -Specify a new `UUID` value for the `requestRollout` to roll out the changes to -the Materialize instance. - -{{< note >}} - -`requestRollout` without the `forcedRollout` field only rolls out if changes -exist to the Materialize instance. To roll out even if there are no changes to -the instance, use with `forcedRollouts`. - -{{< /note >}} - -```shell -# Only rolls out if there are changes -kubectl patch materialize \ - -n \ - --type='merge' \ - -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\"}}" -``` -#### `requestRollout` with `forcedRollouts` - -Specify a new `UUID` value for `forcedRollout` to roll out even when there are -no changes to the instance. Use `forcedRollout` with `requestRollout`. - -```shell -kubectl patch materialize \ - -n materialize-environment \ - --type='merge' \ - -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\", \"forceRollout\": \"$(uuidgen)\"}}" -``` - -### Rollout strategies - -Rollout strategies control how Materialize transitions from the current generation to a new generation during an upgrade. - -The behavior of the new version rollout follows your `rolloutStrategy` setting. - -#### *WaitUntilReady* - ***Default*** - -`WaitUntilReady` creates a new generation of pods and automatically promotes -them as soon as they catch up to the old generation and become `ReadyToPromote`. -Because both generations run simultaneously until the promotion, this strategy -temporarily doubles the required resources to run Materialize. - -{{< note >}} - -While the new generation waits to become `ReadyToPromote`, it runs in a -read-only, un-promoted state and holds back compaction. To prevent it from -sitting in this state indefinitely (which can cause incident-inducing load when -it is eventually promoted), the rollout is bounded by the `rolloutRequestTimeout` -field in the Materialize spec, which defaults to `24h`. - -If the new generation does not become `ReadyToPromote` within -`rolloutRequestTimeout`, the operator cancels the rollout: the new generation is -torn down and the previously-active generation continues serving. You can then -trigger a new rollout by setting `requestRollout` to a new value. - -{{< /note >}} - -#### *ImmediatelyPromoteCausingDowntime* -{{< warning >}} Using the `ImmediatelyPromoteCausingDowntime` rollout flag will cause downtime. {{< /warning >}} - -`ImmediatelyPromoteCausingDowntime` tears down the prior generation, and immediately promotes the new generation without waiting for it to hydrate. This causes downtime until the new generation has hydrated. However, it does not require additional resources. - -#### *ManuallyPromote* - -`ManuallyPromote` allows you to choose when to promote the new generation. This means you can time the promotion for periods when load is low, minimizing the impact of potential downtime for any clients connected to Materialize. This strategy temporarily doubles the required resources to run Materialize. - -To minimize downtime, wait until the new generation has fully hydrated and caught up to the prior generation before promoting. To check hydration status, inspect the `UpToDate` condition in the Materialize resource status. When hydration completes, the condition will be `ReadyToPromote`. - -To promote, update the `forcePromote` field to match the `requestRollout` field in the Materialize spec. If you need to promote before hydration completes, you can set `forcePromote` immediately, but clients may experience downtime. - -{{< warning >}} Leaving a new generation unpromoted for over 6 hours may cause downtime. {{< /warning >}} - -**Do not leave new generations unpromoted indefinitely**. They should either be promoted or canceled. New generations open a read hold on the metadata database that prevents compaction. This hold is only released when the generation is promoted or canceled. If left open too long, promoting or canceling can trigger a spike in deletion load on the metadata database, potentially causing downtime. It is not recommended to leave generations unpromoted for over 6 hours. - -#### *inPlaceRollout* - ***Deprecated*** - -The setting is ignored. - -## Verifying the Upgrade - -After initiating the rollout, you can monitor the status field of the Materialize custom resource to check on the upgrade. - -```shell -# Watch the status of your Materialize environment -kubectl get materialize -n materialize-environment -w - -# Check the logs of the operator -kubectl logs -l app.kubernetes.io/name=materialize-operator -n materialize -``` - -## Cancelling the Upgrade - -You may want to cancel an in-progress rollout if the upgrade has failed. This may be indicated by new pods not being healthy. Before cancelling, verify that the upgrade has not already completed by checking that the deploy generation (found via `status.activeGeneration`) is still the one from before the upgrade. Once an upgrade has already happened, you cannot revert using this method. - -To cancel an in-progress rollout and revert to the last completed rollout -state, revert both `requestRollout` and `environmentdImageRef` back to the -values from the last completed rollout. Reverting `environmentdImageRef` -alongside `requestRollout` keeps the spec aligned with what is actually -running, so a later rollout doesn't accidentally pick up the previously -attempted upgrade image. - -First, retrieve the last completed rollout request ID and the matching -environmentd image ref from your Materialize CR: - -```shell -kubectl get materialize -n materialize-environment \ - -o jsonpath='{.status.lastCompletedRolloutRequest} {.status.lastCompletedRolloutEnvironmentdImageRef}' -``` - -Then, set both fields back to these values in a single patch: - -```shell -kubectl patch materialize \ - -n materialize-environment \ - --type='merge' \ - -p "{\"spec\": {\"requestRollout\": \"\", \"environmentdImageRef\": \"\"}}" -``` diff --git a/doc/user/data/self_managed/materialize_crd_descriptions_v1.json b/doc/user/data/self_managed/materialize_crd_descriptions_v1.json index 6085d5ce6b00e..c410f849e1765 100644 --- a/doc/user/data/self_managed/materialize_crd_descriptions_v1.json +++ b/doc/user/data/self_managed/materialize_crd_descriptions_v1.json @@ -181,7 +181,7 @@ { "name": "rolloutStrategy", "type": "Enum", - "description": "Rollout strategy to use when upgrading this Materialize instance.\n\nValid values:\n- `WaitUntilReady` (default):
        Create a new generation of pods, leaving the old generation around until the\n new ones are ready to take over.\n This minimizes downtime, and is what almost everyone should use.\n- `ManuallyPromote`:
        Create a new generation of pods, leaving the old generation as the serving generation\n until the user manually promotes the new generation.\n \n When using `ManuallyPromote`, the new generation can be promoted at any\n time, even if it has dataflows that are not fully caught up, by setting\n `forcePromote` to the same value as `requestRollout` in the Materialize spec.\n \n To minimize downtime, promotion should occur when the new generation\n has caught up to the prior generation. To determine if the new\n generation has caught up, consult the `UpToDate` condition in the\n status of the Materialize Resource. If the condition's reason is\n `ReadyToPromote` the new generation is ready to promote.\n \n {{}}\n Do not leave new generations unpromoted indefinitely.\n \n The new generation keeps open read holds which prevent compaction. Once promoted or\n cancelled, those read holds are released. If left unpromoted for an extended time, this\n data can build up, and can cause extreme deletion load on the metadata backend database\n when finally promoted or cancelled.\n \n To guard against this, a rollout that remains in progress longer\n than `rolloutRequestTimeout` (default 24h) is automatically\n cancelled.\n {{}}\n- `ImmediatelyPromoteCausingDowntime`:
        {{}}\n THIS WILL CAUSE YOUR MATERIALIZE INSTANCE TO BE UNAVAILABLE FOR SOME TIME!!!\n \n This strategy should ONLY be used by customers with physical hardware who do not have\n enough hardware for the `WaitUntilReady` strategy. If you think you want this, please\n consult with Materialize engineering to discuss your situation.\n {{}}\n \n Tear down the old generation of pods and promote the new generation of pods immediately,\n without waiting for the new generation of pods to be ready.", + "description": "Rollout strategy to use when upgrading this Materialize instance.\n\nValid values:\n- `WaitUntilReady` (default):
        Create a new generation of pods, leaving the old generation around until the\n new ones are ready to take over.\n This minimizes downtime, and is what almost everyone should use.\n- `ManuallyPromote`:
        Create a new generation of pods, leaving the old generation as the serving generation\n until the user manually promotes the new generation.\n \n When using `ManuallyPromote`, the new generation can be promoted at any\n time, even if it has dataflows that are not fully caught up, by setting\n `forcePromote` to the current rollout identifier: in `v1`, the value of\n `status.requestedRolloutHash`; in `v1alpha1`, the `requestRollout` value\n in the spec.\n \n To minimize downtime, promotion should occur when the new generation\n has caught up to the prior generation. To determine if the new\n generation has caught up, consult the `UpToDate` condition in the\n status of the Materialize Resource. If the condition's reason is\n `ReadyToPromote` the new generation is ready to promote.\n \n {{}}\n Do not leave new generations unpromoted indefinitely.\n \n The new generation keeps open read holds which prevent compaction. Once promoted or\n cancelled, those read holds are released. If left unpromoted for an extended time, this\n data can build up, and can cause extreme deletion load on the metadata backend database\n when finally promoted or cancelled.\n \n To guard against this, a rollout that remains in progress longer\n than `rolloutRequestTimeout` (default 24h) is automatically\n cancelled.\n {{}}\n- `ImmediatelyPromoteCausingDowntime`:
        {{}}\n THIS WILL CAUSE YOUR MATERIALIZE INSTANCE TO BE UNAVAILABLE FOR SOME TIME!!!\n \n This strategy should ONLY be used by customers with physical hardware who do not have\n enough hardware for the `WaitUntilReady` strategy. If you think you want this, please\n consult with Materialize engineering to discuss your situation.\n {{}}\n \n Tear down the old generation of pods and promote the new generation of pods immediately,\n without waiting for the new generation of pods to be ready.", "default": "WaitUntilReady", "required": false, "deprecated": false diff --git a/doc/user/data/self_managed/materialize_crd_descriptions_v1alpha1.json b/doc/user/data/self_managed/materialize_crd_descriptions_v1alpha1.json index a87d9961edd19..4d080e4905910 100644 --- a/doc/user/data/self_managed/materialize_crd_descriptions_v1alpha1.json +++ b/doc/user/data/self_managed/materialize_crd_descriptions_v1alpha1.json @@ -189,7 +189,7 @@ { "name": "rolloutStrategy", "type": "Enum", - "description": "Rollout strategy to use when upgrading this Materialize instance.\n\nValid values:\n- `WaitUntilReady` (default):
        Create a new generation of pods, leaving the old generation around until the\n new ones are ready to take over.\n This minimizes downtime, and is what almost everyone should use.\n- `ManuallyPromote`:
        Create a new generation of pods, leaving the old generation as the serving generation\n until the user manually promotes the new generation.\n \n When using `ManuallyPromote`, the new generation can be promoted at any\n time, even if it has dataflows that are not fully caught up, by setting\n `forcePromote` to the same value as `requestRollout` in the Materialize spec.\n \n To minimize downtime, promotion should occur when the new generation\n has caught up to the prior generation. To determine if the new\n generation has caught up, consult the `UpToDate` condition in the\n status of the Materialize Resource. If the condition's reason is\n `ReadyToPromote` the new generation is ready to promote.\n \n {{}}\n Do not leave new generations unpromoted indefinitely.\n \n The new generation keeps open read holds which prevent compaction. Once promoted or\n cancelled, those read holds are released. If left unpromoted for an extended time, this\n data can build up, and can cause extreme deletion load on the metadata backend database\n when finally promoted or cancelled.\n \n To guard against this, a rollout that remains in progress longer\n than `rolloutRequestTimeout` (default 24h) is automatically\n cancelled.\n {{}}\n- `ImmediatelyPromoteCausingDowntime`:
        {{}}\n THIS WILL CAUSE YOUR MATERIALIZE INSTANCE TO BE UNAVAILABLE FOR SOME TIME!!!\n \n This strategy should ONLY be used by customers with physical hardware who do not have\n enough hardware for the `WaitUntilReady` strategy. If you think you want this, please\n consult with Materialize engineering to discuss your situation.\n {{}}\n \n Tear down the old generation of pods and promote the new generation of pods immediately,\n without waiting for the new generation of pods to be ready.", + "description": "Rollout strategy to use when upgrading this Materialize instance.\n\nValid values:\n- `WaitUntilReady` (default):
        Create a new generation of pods, leaving the old generation around until the\n new ones are ready to take over.\n This minimizes downtime, and is what almost everyone should use.\n- `ManuallyPromote`:
        Create a new generation of pods, leaving the old generation as the serving generation\n until the user manually promotes the new generation.\n \n When using `ManuallyPromote`, the new generation can be promoted at any\n time, even if it has dataflows that are not fully caught up, by setting\n `forcePromote` to the current rollout identifier: in `v1`, the value of\n `status.requestedRolloutHash`; in `v1alpha1`, the `requestRollout` value\n in the spec.\n \n To minimize downtime, promotion should occur when the new generation\n has caught up to the prior generation. To determine if the new\n generation has caught up, consult the `UpToDate` condition in the\n status of the Materialize Resource. If the condition's reason is\n `ReadyToPromote` the new generation is ready to promote.\n \n {{}}\n Do not leave new generations unpromoted indefinitely.\n \n The new generation keeps open read holds which prevent compaction. Once promoted or\n cancelled, those read holds are released. If left unpromoted for an extended time, this\n data can build up, and can cause extreme deletion load on the metadata backend database\n when finally promoted or cancelled.\n \n To guard against this, a rollout that remains in progress longer\n than `rolloutRequestTimeout` (default 24h) is automatically\n cancelled.\n {{}}\n- `ImmediatelyPromoteCausingDowntime`:
        {{}}\n THIS WILL CAUSE YOUR MATERIALIZE INSTANCE TO BE UNAVAILABLE FOR SOME TIME!!!\n \n This strategy should ONLY be used by customers with physical hardware who do not have\n enough hardware for the `WaitUntilReady` strategy. If you think you want this, please\n consult with Materialize engineering to discuss your situation.\n {{}}\n \n Tear down the old generation of pods and promote the new generation of pods immediately,\n without waiting for the new generation of pods to be ready.", "default": "WaitUntilReady", "required": false, "deprecated": false diff --git a/doc/user/data/self_managed/upgrades.yml b/doc/user/data/self_managed/upgrades.yml index e36f6aeba6af5..2c2a7d6d1239c 100644 --- a/doc/user/data/self_managed/upgrades.yml +++ b/doc/user/data/self_managed/upgrades.yml @@ -131,12 +131,40 @@ name="upgrade-major-version-restriction" %}} {{< /note >}} +
        +
          +
          +
          + + {{< self-managed/crd-version-note "v1alpha1" >}} + ```shell helm upgrade -n materialize simple-demo materialize/materialize-operator \ -f my-values.yaml \ --version {{< self-managed/versions/get-latest-version >}} ``` +
          +
          + + {{< self-managed/crd-version-note "v1" >}} + + Enabling the `v1` CRD requires cert-manager (or your own certificate) and + API-server ingress to the operator on port `8001`; see the [v26.30 upgrade + notes](/self-managed-deployments/upgrading/#upgrading-to-v2630-and-later-versions) + for prerequisites. + + ```shell + helm upgrade -n materialize simple-demo materialize/materialize-operator \ + -f my-values.yaml \ + --version {{< self-managed/versions/get-latest-version >}} \ + --set operator.args.installV1CRD=true + ``` + +
          +
          +
          + 1. Verify that the Operator is running: ```bash @@ -152,15 +180,22 @@ The **APP VERSION** will be the value that you will use for upgrading Materialize instances. -- name: upgrade-materialize-instance-v1alpha1 +- name: upgrade-materialize-instance content: | **After** you have upgraded your Materialize Operator, upgrade your - Materialize instance(s) to the **APP Version** of the Operator. When - upgrading Materialize Instances versions, changes are not automatically - rolled out by the Operator in order to minimize unexpected downtime and - avoid connection drops at critical periods. Instead, the upgrade process - involves two steps: + Materialize instance(s) to the **APP Version** of the Operator. + +
          +
            +
            +
            + + {{< self-managed/crd-version-note "v1alpha1" >}} + + With `v1alpha1`, changes are not automatically rolled out by the Operator in + order to minimize unexpected downtime and avoid connection drops at critical + periods. Instead, the upgrade process involves two steps: - First, staging the version change to the Materialize custom resource. - Second, rolling out the changes via a `requestRollout` flag. @@ -204,11 +239,11 @@ kubectl -n materialize-environment describe pod -l app=environmentd ``` -- name: upgrade-materialize-instance - content: | +
            +
            - **After** you have upgraded your Materialize Operator, upgrade your - Materialize instance(s) to the **APP Version** of the Operator. + {{< self-managed/crd-version-note "v1" >}} With `v1`, updating the spec automatically triggers a rollout, + so no `requestRollout` is needed. 1. Find the name of the Materialize instance to upgrade. The sample example deployment using the unified Terraform module deploys the Materialize @@ -231,7 +266,7 @@ kubectl patch materialize main \ -n materialize-environment \ --type='merge' \ - -p "{\"apiVersion\": \"materialize.cloud/v1\", \"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\"}}" + -p "{\"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\"}}" ``` 1. Verify the upgrade by checking the `environmentd` events: @@ -239,3 +274,7 @@ ```bash kubectl -n materialize-environment describe pod -l app=environmentd ``` + +
            +
            +
            diff --git a/doc/user/layouts/partials/self-managed/crd-version-note.html b/doc/user/layouts/partials/self-managed/crd-version-note.html new file mode 100644 index 0000000000000..6d610b2bddb64 --- /dev/null +++ b/doc/user/layouts/partials/self-managed/crd-version-note.html @@ -0,0 +1,13 @@ +{{- /* CRD version blurb for tabbed v1alpha1/v1 examples. + Arg: the version string, "v1alpha1" or "v1". */ -}} +{{- $v := . -}} +{{- /* Prepend the docs base path (e.g. /docs) the same way render-link.html + does for markdown links, so the raw resolves under the deployed + subdirectory. */ -}} +{{- $base := strings.TrimRight "/" (urls.Parse site.BaseURL).Path -}} +{{- $adopt := printf "%s/self-managed-deployments/upgrading/#adopting-the-v1-crd" $base -}} +{{- if eq $v "v1alpha1" -}} +

            v1alpha1 is the default CRD version. To get the v1 automatic-rollout behavior, see Adopting the v1 CRD.

            +{{- else -}} +

            The v1 CRD is available starting in v26.30 and is opt-in. See Adopting the v1 CRD to enable it.

            +{{- end -}} diff --git a/doc/user/layouts/shortcodes/self-managed/crd-version-note.html b/doc/user/layouts/shortcodes/self-managed/crd-version-note.html new file mode 100644 index 0000000000000..52b914283f852 --- /dev/null +++ b/doc/user/layouts/shortcodes/self-managed/crd-version-note.html @@ -0,0 +1 @@ +{{- partial "self-managed/crd-version-note.html" (.Get 0) -}} diff --git a/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html b/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html index 249c9b2ec3ca0..1a2e551016a2b 100644 --- a/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html +++ b/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html @@ -3,10 +3,37 @@ 1. Install the Materialize Operator. The operator will be installed in the `materialize` namespace. +
            +
              +
              +
              + + {{ partial "self-managed/crd-version-note.html" "v1alpha1" }} + + ```shell + helm install my-materialize-operator materialize/materialize-operator \ + --namespace=materialize --create-namespace \ + --version {{ $operator_version }} \ + --set observability.podMetrics.enabled=true \ + -f sample-values.yaml + ``` + +
              +
              + + {{ partial "self-managed/crd-version-note.html" "v1" }} + + Enabling the `v1` CRD requires cert-manager, installed in an earlier step. + ```shell helm install my-materialize-operator materialize/materialize-operator \ --namespace=materialize --create-namespace \ --version {{ $operator_version }} \ --set observability.podMetrics.enabled=true \ + --set operator.args.installV1CRD=true \ -f sample-values.yaml ``` + +
              +
              +
              diff --git a/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html b/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html index d9b25385c5ac9..f95a38fe8e24d 100644 --- a/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html +++ b/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html @@ -30,6 +30,13 @@ configuration file. Include any additional configurations that you specify for your deployment. +
              +
                +
                +
                + + {{ partial "self-managed/crd-version-note.html" "v1alpha1" }} + ```shell helm upgrade my-materialize-operator materialize/materialize-operator \ --namespace=materialize \ @@ -38,6 +45,29 @@ --set observability.podMetrics.enabled=true ``` +
                +
                + + {{ partial "self-managed/crd-version-note.html" "v1" }} + + Enabling the `v1` CRD requires cert-manager (or your own certificate) and + API-server ingress to the operator on port `8001`. If you have not already + installed cert-manager, install it first (see the [Install on + kind](/self-managed-deployments/installation/install-on-local-kind/) guide). + + ```shell + helm upgrade my-materialize-operator materialize/materialize-operator \ + --namespace=materialize \ + --version {{ $operator_version }} \ + -f upgrade-values.yaml \ + --set observability.podMetrics.enabled=true \ + --set operator.args.installV1CRD=true + ``` + +
                +
                +
                + 1. Verify that the operator is running: ```bash @@ -66,46 +96,50 @@
                  -
                  +
                  + + {{ partial "self-managed/crd-version-note.html" "v1alpha1" }} | Field | Description | |-------|-------------| | `environmentdImageRef` | Update the version to the new version. This should be the same as the operator version: `{{ $operator_version }}`. | - | `forceRollout`| Enter a new UUID. Can be generated with `uuidgen`.
                  • `forceRollout` triggers a rollout even if no other changes exist.
                  | + | `requestRollout` or `forceRollout`| Enter a new UUID. Can be generated with `uuidgen`.
                  • `requestRollout` triggers a rollout only if changes exist.
                  • `forceRollout` triggers a rollout even if no changes exist.
                  | ```yaml - apiVersion: materialize.cloud/v1 + apiVersion: materialize.cloud/v1alpha1 kind: Materialize metadata: name: 12345678-1234-1234-1234-123456789012 namespace: materialize-environment spec: environmentdImageRef: materialize/environmentd:{{ $environmentd_version }} # Update version - # forceRollout: 33333333-3333-3333-3333-333333333333 # For forced rollouts + requestRollout: 22222222-2222-2222-2222-222222222222 # Enter a new UUID + # forceRollout: 33333333-3333-3333-3333-333333333333 # For forced rollouts rolloutStrategy: WaitUntilReady # The mechanism to use when rolling out the new version. backendSecretName: materialize-backend ```
                  -
                  +
                  + + {{ partial "self-managed/crd-version-note.html" "v1" }} | Field | Description | |-------|-------------| | `environmentdImageRef` | Update the version to the new version. This should be the same as the operator version: `{{ $operator_version }}`. | - | `requestRollout` or `forceRollout`| Enter a new UUID. Can be generated with `uuidgen`.
                  • `requestRollout` triggers a rollout only if changes exist.
                  • `forceRollout` triggers a rollout even if no changes exist.
                  | + | `forceRollout`| Enter a new UUID. Can be generated with `uuidgen`.
                  • `forceRollout` triggers a rollout even if no other changes exist.
                  | ```yaml - apiVersion: materialize.cloud/v1alpha1 + apiVersion: materialize.cloud/v1 kind: Materialize metadata: name: 12345678-1234-1234-1234-123456789012 namespace: materialize-environment spec: environmentdImageRef: materialize/environmentd:{{ $environmentd_version }} # Update version - requestRollout: 22222222-2222-2222-2222-222222222222 # Enter a new UUID - # forceRollout: 33333333-3333-3333-3333-333333333333 # For forced rollouts + # forceRollout: 33333333-3333-3333-3333-333333333333 # For forced rollouts rolloutStrategy: WaitUntilReady # The mechanism to use when rolling out the new version. backendSecretName: materialize-backend ``` diff --git a/doc/user/shared-content/self-managed/upgrade-notes/v26.0.md b/doc/user/shared-content/self-managed/upgrade-notes/v26.0.md index a740e3c7e9224..a59c43a810a08 100644 --- a/doc/user/shared-content/self-managed/upgrade-notes/v26.0.md +++ b/doc/user/shared-content/self-managed/upgrade-notes/v26.0.md @@ -8,7 +8,7 @@ - `ImmediatelyPromoteCausingDowntime` For more information, see - [`rolloutStrategy`](/self-managed-deployments/upgrading/materialize-instances/v1/#rollout-strategies). + [`rolloutStrategy`](/self-managed-deployments/upgrading/#rollout-strategies). - New requirements were introduced for [license keys](/releases/#license-key). To upgrade, you will first need to add a license key to the `backendSecret` diff --git a/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md b/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md index eda81b9f67213..ad5ca0f9abd26 100644 --- a/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md +++ b/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md @@ -31,7 +31,7 @@ spec: With the new change, the `requestRollout` field will be removed, along with all previously deprecated fields. -Switching to `v1` is **opt-in**. You may continue to apply `v1alpha1` CRs and your existing instances will behave exactly as before. We recommend you opt-in to `v1` at your convenience, as we will migrate to the `v1` behavior in the next major release. For step-by-step migration instructions, see [Switching from v1alpha1 to v1](/self-managed-deployments/upgrading/materialize-instances/#switching-from-v1alpha1-to-v1). +Adopting `v1` is **opt-in**. You may continue to apply `v1alpha1` CRs and your existing instances will behave exactly as before. We recommend you opt-in to `v1` at your convenience, as we will migrate to the `v1` behavior in the next major release. For step-by-step instructions, see [Adopting the v1 CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). {{< important >}} @@ -62,7 +62,7 @@ the required infrastructure changes (cert-manager and network ingress) will be handled for you automatically. To upgrade, update each module's `source` to point to the new release tag and -run `terraform init -upgrade && terraform plan && terraform apply`. To enable +run `terraform init && terraform plan && terraform apply`. To enable the `v1` CRD, also set the Helm value `operator.args.installV1CRD=true` in the values passed to the operator module. diff --git a/src/cloud-resources/src/crd/materialize.rs b/src/cloud-resources/src/crd/materialize.rs index 483ca37f8b915..2729cef44e82a 100644 --- a/src/cloud-resources/src/crd/materialize.rs +++ b/src/cloud-resources/src/crd/materialize.rs @@ -45,7 +45,9 @@ pub enum MaterializeRolloutStrategy { /// /// When using `ManuallyPromote`, the new generation can be promoted at any /// time, even if it has dataflows that are not fully caught up, by setting - /// `forcePromote` to the same value as `requestRollout` in the Materialize spec. + /// `forcePromote` to the current rollout identifier: in `v1`, the value of + /// `status.requestedRolloutHash`; in `v1alpha1`, the `requestRollout` value + /// in the spec. /// /// To minimize downtime, promotion should occur when the new generation /// has caught up to the prior generation. To determine if the new From b279f044aac6db3cbf5f8f142ccc49efe3b36014 Mon Sep 17 00:00:00 2001 From: kay-kim Date: Thu, 25 Jun 2026 14:46:39 -0400 Subject: [PATCH 3/6] Tweaks to install and upgrade guides --- .../installation/install-on-aws.md | 17 ++++++--- .../installation/install-on-azure.md | 20 +++++++---- .../installation/install-on-gcp.md | 7 +++- .../installation/install-on-local-kind.md | 13 ++++--- .../upgrading/upgrade-on-azure.md | 2 +- .../upgrading/upgrade-on-kind.md | 4 +-- doc/user/data/self_managed/installation.yml | 11 +++++- doc/user/data/self_managed/upgrades.yml | 31 +++++++++------- .../self-managed/crd-version-note.html | 2 +- ...stall-helm-version-local-kind-install.html | 35 ++++++------------- .../upgrade/upgrade-steps-local-kind.html | 28 +++++++++------ 11 files changed, 99 insertions(+), 71 deletions(-) diff --git a/doc/user/content/self-managed-deployments/installation/install-on-aws.md b/doc/user/content/self-managed-deployments/installation/install-on-aws.md index 90e0abb408ca1..bc575027eefd3 100644 --- a/doc/user/content/self-managed-deployments/installation/install-on-aws.md +++ b/doc/user/content/self-managed-deployments/installation/install-on-aws.md @@ -156,17 +156,24 @@ authentication mechanisms. 1. Create a `terraform.tfvars` file with the following variables: - - `name_prefix`: Prefix for all resource names (e.g., `simple-demo`) - - `aws_region`: AWS region for deployment (e.g., `us-east-1`) - - `aws_profile`: AWS CLI profile to use - - `license_key`: Materialize license key - - `tags`: Map of tags to apply to resources + | Variable | Description | + | ----------- | ----------------------------| + | `name_prefix` | Prefix for all resource names (e.g., `simple-demo`). | + | `aws_region` | AWS region for deployment (e.g., `us-east-1`). | + | `aws_profile` | AWS CLI profile to use. | + | `license_key` | Materialize license key. | + | `crd_version` | CRD API version to use for the Materialize instance: `v1` (recommended) or `v1alpha1`. | + | `tags` | Map of tags to apply to resources. | + + {{% include-from-yaml data="self_managed/installation" + name="installation-tfvars-crd-version-tip" %}} ```hcl name_prefix = "simple-demo" aws_region = "us-east-1" aws_profile = "your-aws-profile" license_key = "your-materialize-license-key" + crd_version = "v1" # Recommended. Default is v1alpha1; v1 is available starting in Materialize v26.30. tags = { environment = "demo" } diff --git a/doc/user/content/self-managed-deployments/installation/install-on-azure.md b/doc/user/content/self-managed-deployments/installation/install-on-azure.md index 199c6ddb8c2fa..32bdd6bf5a3f9 100644 --- a/doc/user/content/self-managed-deployments/installation/install-on-azure.md +++ b/doc/user/content/self-managed-deployments/installation/install-on-azure.md @@ -173,13 +173,18 @@ authentication mechanisms. 1. Create a `terraform.tfvars` file with the following variables: - - `subscription_id`: Azure subscription ID - - `resource_group_name`: Name for the resource group to create (e.g. - `mz-demo-rg`) - - `name_prefix`: Prefix for all resource names (e.g., `simple-demo`) - - `location`: Azure region for deployment (e.g., `westus2`) - - `license_key`: Materialize license key - - `tags`: Map of tags to apply to resources + | Variable | Description | + | ----------- | ----------------------------| + | `subscription_id` | Azure subscription ID. | + | `resource_group_name` | Name for the resource group to create (e.g., `mz-demo-rg`). | + | `name_prefix` | Prefix for all resource names (e.g., `simple-demo`). | + | `location` | Azure region for deployment (e.g., `westus2`). | + | `license_key` | Materialize license key. | + | `crd_version` | CRD API version to use for the Materialize instance: `v1` (recommended) or `v1alpha1`. | + | `tags` | Map of tags to apply to resources. | + + {{% include-from-yaml data="self_managed/installation" + name="installation-tfvars-crd-version-tip" %}} ```hcl subscription_id = "your-subscription-id" @@ -187,6 +192,7 @@ authentication mechanisms. name_prefix = "simple-demo" location = "westus2" license_key = "your-materialize-license-key" + crd_version = "v1" # Recommended. Default is v1alpha1; v1 is available starting in Materialize v26.30. tags = { environment = "demo" } diff --git a/doc/user/content/self-managed-deployments/installation/install-on-gcp.md b/doc/user/content/self-managed-deployments/installation/install-on-gcp.md index 166d63b2c8738..0106655ba34a1 100644 --- a/doc/user/content/self-managed-deployments/installation/install-on-gcp.md +++ b/doc/user/content/self-managed-deployments/installation/install-on-gcp.md @@ -187,13 +187,18 @@ authentication mechanisms. | `name_prefix` | Set a prefix for all resource names (e.g., `simple-demo`) as well as your release name for the Operator | | `region` | Set the GCP region for the deployment (e.g., `us-central1`). | | `license_key` | Set to your Materialize license key. | + | `crd_version` | Set the CRD API version to use for the Materialize instance: `v1` (recommended) or `v1alpha1`. | | `labels` | Set to the labels to apply to resources. | - ```bash + {{% include-from-yaml data="self_managed/installation" + name="installation-tfvars-crd-version-tip" %}} + + ```hcl project_id = "my-gcp-project" name_prefix = "simple-demo" region = "us-central1" license_key = "your-materialize-license-key" + crd_version = "v1" # Recommended. Default is v1alpha1; v1 is available starting in Materialize v26.30. labels = { environment = "demo" created_by = "terraform" diff --git a/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md b/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md index 1bce6a438ebbf..2bf6b06077084 100644 --- a/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md +++ b/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md @@ -107,13 +107,12 @@ Starting in v26.0, Self-Managed Materialize requires a license key. kubectl get nodes --show-labels ``` -1. Install cert-manager - - Cert-manager is used for generating TLS certificates needed by the materialize operator - for CRD conversion webhooks. It is currently only required if you enable the v1 - version of the Materialize CRD by setting `operator.args.installV1CRD=true` - when installing the operator, but certificates will become required in a - future version of Materialize. +1. Install `cert-manager`. `cert-manager` is + used for generating TLS certificates needed by the Materialize operator for + CRD conversion webhooks. It is only required if you enable the `v1` version + of the Materialize CRD when installing the operator. However, we recommend + installing it even if you are not yet enabling `v1` to simplify the future + transition. ```shell helm install cert-manager oci://quay.io/jetstack/charts/cert-manager \ diff --git a/doc/user/content/self-managed-deployments/upgrading/upgrade-on-azure.md b/doc/user/content/self-managed-deployments/upgrading/upgrade-on-azure.md index 291826d42483e..e406839107a14 100644 --- a/doc/user/content/self-managed-deployments/upgrading/upgrade-on-azure.md +++ b/doc/user/content/self-managed-deployments/upgrading/upgrade-on-azure.md @@ -66,7 +66,7 @@ The following procedure performs a rolling upgrade, where both the old and new M - `` with your cluster name; i.e., the `aks_cluster_name` in the Terraform output. For the sample example, - your cluster name has the form `{prefix_name}-aks`; e.g., simple-demo-aks`. + your cluster name has the form `{prefix_name}-aks`; e.g., `simple-demo-aks`. ```bash # az aks get-credentials --resource-group --name diff --git a/doc/user/content/self-managed-deployments/upgrading/upgrade-on-kind.md b/doc/user/content/self-managed-deployments/upgrading/upgrade-on-kind.md index 866e8a933b2b6..943c8ac17c8cd 100644 --- a/doc/user/content/self-managed-deployments/upgrading/upgrade-on-kind.md +++ b/doc/user/content/self-managed-deployments/upgrading/upgrade-on-kind.md @@ -29,7 +29,7 @@ If you don't have Helm version 3.2.0+ installed, install. For details, see the ### `kubectl` This tutorial uses `kubectl`. To install, refer to the [`kubectl` -documentationq](https://kubernetes.io/docs/tasks/tools/). +documentation](https://kubernetes.io/docs/tasks/tools/). For help with `kubectl` commands, see [kubectl Quick reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). @@ -43,7 +43,7 @@ reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). {{< important >}} The following procedure performs a rolling upgrade, where both the old and new -Materialize instances are running before the the old instance are removed. +Materialize instances are running before the old instance are removed. When performing a rolling upgrade, ensure you have enough resources to support having both the old and new Materialize instances running. diff --git a/doc/user/data/self_managed/installation.yml b/doc/user/data/self_managed/installation.yml index 8c6bb711d774d..390e5fe453249 100644 --- a/doc/user/data/self_managed/installation.yml +++ b/doc/user/data/self_managed/installation.yml @@ -86,9 +86,18 @@ If you run into an error during deployment, refer to the [Troubleshooting](/self-managed-deployments/troubleshooting/). +- name: installation-tfvars-crd-version-tip + content: | + {{< tip >}} + CRD API version `v1` is available starting in Materialize v26.30 and + Materialize Terraform module version v3.0.15. + For earlier versions, or if you want to remain on the default `v1alpha1`, + comment out the `crd_version` line or set it to `v1alpha1`. + {{< /tip >}} + - name: installation-tfvars-variables-optional content: | - **Optional variables**: + **Additional variables**: - `internal_load_balancer`: Flag that determines whether the load balancer is internal (default) or public. diff --git a/doc/user/data/self_managed/upgrades.yml b/doc/user/data/self_managed/upgrades.yml index 2c2a7d6d1239c..8472865bd8a69 100644 --- a/doc/user/data/self_managed/upgrades.yml +++ b/doc/user/data/self_managed/upgrades.yml @@ -131,10 +131,17 @@ name="upgrade-major-version-restriction" %}} {{< /note >}} + Select the tab for the CRD API version your deployment currently uses. + This step does not cover migrating from `v1alpha1` to `v1`; to migrate, + see [Adopting the v1 + CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). (The `v1` + CRD API version is available starting in Materialize v26.30 / Terraform + module v3.0.15.) +
                    -
                    +
                    {{< self-managed/crd-version-note "v1alpha1" >}} @@ -145,14 +152,7 @@ ```
                    -
                    - - {{< self-managed/crd-version-note "v1" >}} - - Enabling the `v1` CRD requires cert-manager (or your own certificate) and - API-server ingress to the operator on port `8001`; see the [v26.30 upgrade - notes](/self-managed-deployments/upgrading/#upgrading-to-v2630-and-later-versions) - for prerequisites. +
                    ```shell helm upgrade -n materialize simple-demo materialize/materialize-operator \ @@ -186,10 +186,17 @@ **After** you have upgraded your Materialize Operator, upgrade your Materialize instance(s) to the **APP Version** of the Operator. + Select the tab for the CRD API version your deployment currently uses. This + step does not cover migrating from `v1alpha1` to `v1`; to migrate, see + [Adopting the v1 + CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). (The `v1` + CRD API version is available starting in Materialize v26.30 / Terraform + module v3.0.15.) +
                      -
                      +
                      {{< self-managed/crd-version-note "v1alpha1" >}} @@ -240,9 +247,9 @@ ```
                      -
                      +
                      - {{< self-managed/crd-version-note "v1" >}} With `v1`, updating the spec automatically triggers a rollout, + With `v1`, updating the spec automatically triggers a rollout, so no `requestRollout` is needed. 1. Find the name of the Materialize instance to upgrade. The sample example diff --git a/doc/user/layouts/partials/self-managed/crd-version-note.html b/doc/user/layouts/partials/self-managed/crd-version-note.html index 6d610b2bddb64..b2d5023fabb42 100644 --- a/doc/user/layouts/partials/self-managed/crd-version-note.html +++ b/doc/user/layouts/partials/self-managed/crd-version-note.html @@ -7,7 +7,7 @@ {{- $base := strings.TrimRight "/" (urls.Parse site.BaseURL).Path -}} {{- $adopt := printf "%s/self-managed-deployments/upgrading/#adopting-the-v1-crd" $base -}} {{- if eq $v "v1alpha1" -}} -

                      v1alpha1 is the default CRD version. To get the v1 automatic-rollout behavior, see Adopting the v1 CRD.

                      +

                      v1alpha1 is the default CRD version. With v1alpha1, rollouts require manually rotating a UUID. For a simplified rollout behavior, see Adopting the v1 CRD.

                      {{- else -}}

                      The v1 CRD is available starting in v26.30 and is opt-in. See Adopting the v1 CRD to enable it.

                      {{- end -}} diff --git a/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html b/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html index 1a2e551016a2b..c40783133131a 100644 --- a/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html +++ b/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html @@ -3,27 +3,14 @@ 1. Install the Materialize Operator. The operator will be installed in the `materialize` namespace. -
                      -
                        -
                        -
                        - - {{ partial "self-managed/crd-version-note.html" "v1alpha1" }} - - ```shell - helm install my-materialize-operator materialize/materialize-operator \ - --namespace=materialize --create-namespace \ - --version {{ $operator_version }} \ - --set observability.podMetrics.enabled=true \ - -f sample-values.yaml - ``` - -
                        -
                        - - {{ partial "self-managed/crd-version-note.html" "v1" }} - - Enabling the `v1` CRD requires cert-manager, installed in an earlier step. + Starting in v26.30, you can choose between CRD versions + [`v1`](/self-managed-deployments/upgrading/#upgrading-to-v2630-and-later-versions) + and `v1alpha1`. The following enables the `v1` CRD version by setting + `operator.args.installV1CRD=true`. The `v1` CRD provides a [simplified + rollout + behavior](/self-managed-deployments/upgrading/#upgrading-to-v2630-and-later-versions), + but requires `cert-manager`, installed in an [earlier + step](#install-cert-manager) in this guide. ```shell helm install my-materialize-operator materialize/materialize-operator \ @@ -34,6 +21,6 @@ -f sample-values.yaml ``` -
                        -
                        -
                        + To remain on the default `v1alpha1` CRD version, omit `--set + operator.args.installV1CRD=true`. With `v1alpha1`, rollouts require manually + rotating a UUID. diff --git a/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html b/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html index f95a38fe8e24d..4873060dcc76d 100644 --- a/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html +++ b/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html @@ -30,10 +30,16 @@ configuration file. Include any additional configurations that you specify for your deployment. + Select the tab for the CRD API version your deployment currently uses. This + step does not cover migrating from `v1alpha1` to `v1`; to migrate, see + [Adopting the v1 + CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). (The `v1` CRD + API version is available starting in Materialize v26.30.) +
                          -
                          +
                          {{ partial "self-managed/crd-version-note.html" "v1alpha1" }} @@ -46,9 +52,7 @@ ```
                          -
                          - - {{ partial "self-managed/crd-version-note.html" "v1" }} +
                          Enabling the `v1` CRD requires cert-manager (or your own certificate) and API-server ingress to the operator on port `8001`. If you have not already @@ -93,10 +97,16 @@ 1. Create a new `upgrade-materialize.yaml` file, updating the following fields: + Select the tab for the CRD API version your deployment currently uses. This + step does not cover migrating from `v1alpha1` to `v1`; to migrate, see + [Adopting the v1 + CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). (The `v1` CRD + API version is available starting in Materialize v26.30.) +
                            -
                            +
                            {{ partial "self-managed/crd-version-note.html" "v1alpha1" }} @@ -121,14 +131,12 @@ ```
                            -
                            - - {{ partial "self-managed/crd-version-note.html" "v1" }} +
                            | Field | Description | |-------|-------------| - | `environmentdImageRef` | Update the version to the new version. This should be the same as the operator version: `{{ $operator_version }}`. | - | `forceRollout`| Enter a new UUID. Can be generated with `uuidgen`.
                            • `forceRollout` triggers a rollout even if no other changes exist.
                            | + | `environmentdImageRef` | Update the version to the new version. This should be the same as the operator version: `{{ $operator_version }}`. Updating this field automatically triggers a rollout. | + | `forceRollout` | **Optional.** Set to a new UUID (can be generated with `uuidgen`) only to force a rollout when no other changes exist. | ```yaml From 4127a12441b552fe38522ba89a66dbf0f87da244 Mon Sep 17 00:00:00 2001 From: Kay Kim Date: Thu, 25 Jun 2026 17:20:15 -0400 Subject: [PATCH 4/6] Update doc/user/content/self-managed-deployments/upgrading/upgrade-on-kind.md Co-authored-by: val-materialize --- .../self-managed-deployments/upgrading/upgrade-on-kind.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user/content/self-managed-deployments/upgrading/upgrade-on-kind.md b/doc/user/content/self-managed-deployments/upgrading/upgrade-on-kind.md index 943c8ac17c8cd..9e169fd470b32 100644 --- a/doc/user/content/self-managed-deployments/upgrading/upgrade-on-kind.md +++ b/doc/user/content/self-managed-deployments/upgrading/upgrade-on-kind.md @@ -43,7 +43,7 @@ reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). {{< important >}} The following procedure performs a rolling upgrade, where both the old and new -Materialize instances are running before the old instance are removed. +Materialize instances are running before the old instances are removed. When performing a rolling upgrade, ensure you have enough resources to support having both the old and new Materialize instances running. From 183b8fb0a555ce2d517359c61d94c08a611cf366 Mon Sep 17 00:00:00 2001 From: kay-kim Date: Thu, 25 Jun 2026 20:37:25 -0400 Subject: [PATCH 5/6] small reorg --- doc/user/content/releases/_index.md | 4 +- .../installation/install-on-local-kind.md | 2 +- .../_index.md | 4 +- .../materialize-crd-field-descriptions/v1.md | 8 +- .../v1alpha1.md | 8 +- .../upgrading/_index.md | 159 ++++-------------- .../upgrading/adopting-the-v1-crd.md | 102 +++++++++++ .../upgrading/version-notes.md | 29 ++++ doc/user/data/self_managed/upgrades.yml | 13 +- .../self-managed/crd-version-note.html | 2 +- ...stall-helm-version-local-kind-install.html | 4 +- .../upgrade/upgrade-steps-local-kind.html | 4 +- .../general-rules-for-upgrades.md | 4 +- .../self-managed/upgrade-notes/v26.30.md | 4 +- 14 files changed, 197 insertions(+), 150 deletions(-) create mode 100644 doc/user/content/self-managed-deployments/upgrading/adopting-the-v1-crd.md create mode 100644 doc/user/content/self-managed-deployments/upgrading/version-notes.md diff --git a/doc/user/content/releases/_index.md b/doc/user/content/releases/_index.md index bb9953890e8d9..8ee4c8cc52105 100644 --- a/doc/user/content/releases/_index.md +++ b/doc/user/content/releases/_index.md @@ -1422,8 +1422,8 @@ See also Upgrade Notes for release specific notes. {{< include-md file="shared-content/self-managed/upgrade-notes/v26.0.md" >}} -See also [Version specific upgrade -notes](/self-managed-deployments/upgrading/#version-specific-upgrade-notes). +See also [Version-specific upgrade +notes](/self-managed-deployments/upgrading/version-notes/). ## See also diff --git a/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md b/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md index 2bf6b06077084..9d34874c3c8d6 100644 --- a/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md +++ b/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md @@ -291,7 +291,7 @@ Starting in v26.0, Self-Managed Materialize requires a license key. operator, edit `sample-materialize.yaml` to set `apiVersion: materialize.cloud/v1` and remove any `requestRollout` field before applying. See [Adopting the v1 - CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). + CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). {{< /tip >}} 1. Verify the installation and check the status: diff --git a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/_index.md b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/_index.md index ddd5303da8d57..e140b620c7a64 100644 --- a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/_index.md +++ b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/_index.md @@ -13,5 +13,5 @@ aliases: Select the CRD API version for your Materialize deployment: -- [v1 (available starting v26.30+)](/self-managed-deployments/materialize-crd-field-descriptions/v1/) -- [v1alpha1 (before v26.30)](/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1/) +- [v1 (Available starting v26.30+)](/self-managed-deployments/materialize-crd-field-descriptions/v1/) +- [v1alpha1 (*default*)](/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1/) diff --git a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md index 5141a555fd1c2..b1ddd51bf2f03 100644 --- a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md +++ b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md @@ -8,10 +8,10 @@ menu: --- {{< note >}} -The `v1` CRD is available starting in v26.30 and is opt-in. To get the new -automatic-rollout behavior for an existing instance, see [Adopting the v1 -CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). -For the earlier API, see [v1alpha1](/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1/). +The `v1` CRD is available starting in v26.30 and is opt-in. If you are on +`v1alpha1` and want a simplified rollout behavior of `v1`, see [Adopting the v1 +CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). +For the `v1alpha1` field descriptions, see [CRD v1alpha1 field descriptions](/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1/). {{}} {{% self-managed/materialize-crd-descriptions-v1 %}} diff --git a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md index 3ebe6dd703642..a08d254cb27fb 100644 --- a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md +++ b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md @@ -8,10 +8,12 @@ menu: --- {{< note >}} -`v1alpha1` is the default CRD version (used before v26.30 and still the default). +`v1alpha1` is the default CRD version (default). With v1alpha1, rollouts require manually rotating a UUID. Starting in v26.30, the [v1](/self-managed-deployments/materialize-crd-field-descriptions/v1/) -CRD is available; to get its automatic-rollout behavior, see [Adopting the v1 -CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). +CRD is available and provides a simplified rollout behavior. + +To switch to `v1`, see [Adopting the v1 +CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). {{}} {{% self-managed/materialize-crd-descriptions-v1alpha1 %}} diff --git a/doc/user/content/self-managed-deployments/upgrading/_index.md b/doc/user/content/self-managed-deployments/upgrading/_index.md index de267f6dee9b7..fc50f137ddc2e 100644 --- a/doc/user/content/self-managed-deployments/upgrading/_index.md +++ b/doc/user/content/self-managed-deployments/upgrading/_index.md @@ -93,105 +93,23 @@ helm upgrade -n materialize my-demo materialize/operator \ ## Upgrading Materialize Instances -After upgrading the operator, upgrade each Materialize instance to the operator's -**App Version** by updating its `environmentdImageRef`. For step-by-step -instructions, use the [upgrade guide](#upgrade-guides) for your environment -(Kind, AWS, Azure, or GCP). This section describes the rollout behavior those -guides rely on. +After upgrading the operator, upgrade each Materialize instance to the +operator's **App Version** by updating its `environmentdImageRef`. For +step-by-step instructions, use the [upgrade guide](#upgrade-guides) for your +environment (Kind, AWS, Azure, or GCP). This section explains how instance +rollouts work. -### CRD API versions - -Starting in v26.30, the Materialize Operator supports two CRD API versions: - -- **v1alpha1** (default) uses a two-step rollout: first stage the spec change, - then trigger a rollout with a new `requestRollout` UUID. -- **v1** simplifies upgrades — rollouts trigger automatically when spec fields - change, removing the need to manually set a `requestRollout` UUID. - -Adopting v1 is **opt-in** for now. Upgrading the operator to v26.30+ does not -change your existing v1alpha1 CRs or their behavior; you can continue using -v1alpha1 and adopt v1 for individual instances at your own pace. - -{{< important >}} -In the next major release, all Materialize CRs will be force upgraded to v1. You -will still be able to apply v1alpha1 CRs, but they will be auto-converted to v1 -and use the v1 rollout behavior. We recommend opting in to v1 at your convenience -to migrate on your own schedule before the upgrade is mandatory. -{{< /important >}} - -## Adopting the v1 CRD - -Adopting v1 is opt-in and does not trigger a rollout on its own. Before you -begin, ensure you have completed the prerequisites in the [v26.30 upgrade -notes](#upgrading-to-v2630-and-later-versions) (cert-manager, network/firewall -changes), and have enabled the v1 CRD by setting the Helm value -`operator.args.installV1CRD=true` on the operator. Without this value, the -operator only installs the v1alpha1 CRD version, and the Kubernetes API server -rejects v1 CRs. - -### How it works - -The v1alpha1 CRD remains the storage version. When you submit a v1 CR, the -operator's conversion webhook automatically converts it to v1alpha1 for storage. -During conversion, the webhook computes a SHA256 hash of a subset of the spec -fields and derives a deterministic `requestRollout` UUID from it. The hash covers -the fields that affect the running `environmentd` (for example, -`environmentdImageRef`, `environmentdExtraArgs`, `environmentdExtraEnv`, resource -requirements, `podAnnotations`, `podLabels`, `authenticatorKind`, `enableRbac`, -`rolloutStrategy`, and `forceRollout`). It excludes fields that do not require a -rollout, such as `balancerd`/`console` resource requirements and replica counts. -The operator triggers a rollout whenever `requestRollout` differs from the last -completed rollout, so: - -- **Adopting v1 on an existing v1alpha1 instance typically triggers one - rollout.** The derived `requestRollout` is computed from the spec hash and will - not match the `requestRollout` you previously set by hand, so the instance - rolls out once even if nothing else in the spec changed. Adopt v1 during a - window where a rollout is acceptable. -- **Once on v1, an unchanged spec does not trigger a rollout.** Reapplying the - same spec produces the same hash and the same derived `requestRollout`. - Changing a hashed spec field produces a new value and triggers a rollout - automatically. - -### Using kubectl - -To adopt v1 for an existing instance, apply your CR with `apiVersion: -materialize.cloud/v1` and remove the `requestRollout` field: - -```shell -kubectl apply -f - < - namespace: -spec: - environmentdImageRef: - backendSecretName: - # ... other spec fields (copy from your existing CR, removing requestRollout) -EOF -``` - -### Using Terraform - -If you are managing your Materialize instance with the [Materialize Terraform -modules](https://github.com/MaterializeInc/materialize-terraform-self-managed), -set: - -```hcl -crd_version = "v1" -request_rollout = null -``` - -### Returning to the v1alpha1 behavior +## Rollout configuration -You can go back to the v1alpha1 rollout behavior at any time by applying your CR -with `apiVersion: materialize.cloud/v1alpha1` and an explicit `requestRollout` -UUID. +Materialize supports two CRD API versions: `v1alpha1` (default) and `v1` +(available starting in v26.30). -## Rollout configuration +How you trigger a rollout depends on the CRD API version of your instances. -How you trigger a rollout depends on your CRD API version. +{{< tip >}} +To migrate a `v1alpha1` instance to `v1`, see +[Adopting the v1 CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). +{{< /tip >}} {{< tabs >}} {{< tab "v1alpha1" >}} @@ -226,22 +144,25 @@ kubectl patch materialize \ {{< /tab >}} {{< tab "v1" >}} -With v1, the operator computes a hash of the spec fields and automatically -triggers a rollout when the hash changes — there is no `requestRollout` field. +With v1, updating the spec automatically triggers a rollout and there is no +`requestRollout` field. For details on the underlying mechanism, see [How it +works](/self-managed-deployments/upgrading/adopting-the-v1-crd/#how-it-works). -Specify a new `UUID` value for `forceRollout` to trigger a rollout even when -there are no other changes to the instance. The `forceRollout` value is included -in the hash, so changing it produces a new hash and triggers a rollout. Set it in -your manifest and reapply: +To trigger a rollout even when there are no other changes to the instance, +specify a new `UUID` value for `forceRollout`. That is: -```yaml -spec: - forceRollout: # e.g. the output of `uuidgen` -``` +- Set it in your manifest: -```shell -kubectl apply -f materialize.yaml -``` + ```yaml + spec: + forceRollout: # e.g. the output of `uuidgen` + ``` + +- Then reapply. + + ```shell + kubectl apply -f materialize.yaml + ``` {{< /tab >}} {{< /tabs >}} @@ -376,25 +297,13 @@ kubectl apply -f previous_materialize_configuration.yaml {{< /tab >}} {{< /tabs >}} -## Version Specific Upgrade Notes - -### Upgrading to `v26.30` and later versions - -{{< include-md file="shared-content/self-managed/upgrade-notes/v26.30.md" >}} - -### Upgrading to `v26.1` and later versions - -{{< include-md file="shared-content/self-managed/upgrade-notes/v26.1.md" >}} - -### Upgrading to `v26.0` - -{{< include-md file="shared-content/self-managed/upgrade-notes/v26.0.md" >}} +## See also -### Upgrading between minor versions less than `v26` - - Prior to `v26`, you must upgrade at most one minor version at a time. For - example, upgrading from `v25.1.5` to `v25.2.16` is permitted. +- [Version-specific upgrade + notes](/self-managed-deployments/upgrading/version-notes/) -## See also +- [Adopting the v1 + CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/) - [Materialize Operator Configuration](/self-managed-deployments/operator-configuration/) diff --git a/doc/user/content/self-managed-deployments/upgrading/adopting-the-v1-crd.md b/doc/user/content/self-managed-deployments/upgrading/adopting-the-v1-crd.md new file mode 100644 index 0000000000000..4e8be4583ee22 --- /dev/null +++ b/doc/user/content/self-managed-deployments/upgrading/adopting-the-v1-crd.md @@ -0,0 +1,102 @@ +--- +title: "Adopting the v1 CRD" +description: "Adopt the v1 Materialize CRD API version for Self-Managed Materialize." +menu: + main: + parent: "upgrading" + weight: 80 +--- + +This page describes the Materialize CRD API versions and how to adopt `v1` for +your Materialize instances. + +## CRD API versions + +Starting in v26.30, the Materialize Operator supports two CRD API versions: + +- **v1alpha1** (default) uses a two-step rollout: first stage the spec change, + then trigger a rollout with a new `requestRollout` UUID. +- **v1** simplifies upgrades — rollouts trigger automatically when spec fields + change, removing the need to manually set a `requestRollout` UUID. + +Adopting v1 is **opt-in** for now. Upgrading the operator to v26.30+ does not +change your existing v1alpha1 CRs or their behavior; you can continue using +v1alpha1 and adopt v1 for individual instances at your own pace. + +{{< important >}} +In the next major release, all Materialize CRs will be force upgraded to v1. You +will still be able to apply v1alpha1 CRs, but they will be auto-converted to v1 +and use the v1 rollout behavior. We recommend opting in to v1 at your convenience +to migrate on your own schedule before the upgrade is mandatory. +{{< /important >}} + +## Prerequisites + +Before you begin, ensure you have completed the prerequisites in the [v26.30 +upgrade +notes](/self-managed-deployments/upgrading/version-notes/#upgrading-to-v2630-and-later-versions) +(cert-manager, network/firewall changes), and have enabled the v1 CRD by setting +the Helm value `operator.args.installV1CRD=true` on the operator. Enabling the +v1 CRD alone does not roll out your existing instances. Without this value, the +operator only installs the v1alpha1 CRD version, and the Kubernetes API server +rejects v1 CRs. + +## How it works + +The v1alpha1 CRD remains the storage version. When you submit a v1 CR, the +operator's conversion webhook automatically converts it to v1alpha1 for storage. +During conversion, the webhook computes a SHA256 hash of a subset of the spec +fields and derives a deterministic `requestRollout` UUID from it. The hash covers +the fields that affect the running `environmentd` (for example, +`environmentdImageRef`, `environmentdExtraArgs`, `environmentdExtraEnv`, resource +requirements, `podAnnotations`, `podLabels`, `authenticatorKind`, `enableRbac`, +`rolloutStrategy`, and `forceRollout`). It excludes fields that do not require a +rollout, such as `balancerd`/`console` resource requirements and replica counts. +The operator triggers a rollout whenever `requestRollout` differs from the last +completed rollout, so: + +- **Adopting v1 on an existing v1alpha1 instance typically triggers one + rollout.** The derived `requestRollout` is computed from the spec hash and will + not match the `requestRollout` you previously set by hand, so the instance + rolls out once even if nothing else in the spec changed. Adopt v1 during a + window where a rollout is acceptable. +- **Once on v1, an unchanged spec does not trigger a rollout.** Reapplying the + same spec produces the same hash and the same derived `requestRollout`. + Changing a hashed spec field produces a new value and triggers a rollout + automatically. + +## Using kubectl + +To adopt v1 for an existing instance, apply your CR with `apiVersion: +materialize.cloud/v1` and remove the `requestRollout` field: + +```shell +kubectl apply -f - < + namespace: +spec: + environmentdImageRef: + backendSecretName: + # ... other spec fields (copy from your existing CR, removing requestRollout) +EOF +``` + +## Using Terraform + +If you are managing your Materialize instance with the [Materialize Terraform +modules](https://github.com/MaterializeInc/materialize-terraform-self-managed), +set: + +```hcl +crd_version = "v1" +request_rollout = null +``` + +## Returning to the v1alpha1 behavior + +You can go back to the v1alpha1 rollout behavior at any time by applying your CR +with `apiVersion: materialize.cloud/v1alpha1` and an explicit `requestRollout` +UUID. diff --git a/doc/user/content/self-managed-deployments/upgrading/version-notes.md b/doc/user/content/self-managed-deployments/upgrading/version-notes.md new file mode 100644 index 0000000000000..e9005df8b8bbb --- /dev/null +++ b/doc/user/content/self-managed-deployments/upgrading/version-notes.md @@ -0,0 +1,29 @@ +--- +title: "Version-specific upgrade notes" +description: "Version-specific notes for upgrading Self-Managed Materialize." +menu: + main: + parent: "upgrading" + weight: 5 +--- + +Review the notes for your target version before upgrading. For the general +upgrade procedure, see the [upgrade +guides](/self-managed-deployments/upgrading/#upgrade-guides). + +## Upgrading to `v26.30` and later versions + +{{< include-md file="shared-content/self-managed/upgrade-notes/v26.30.md" >}} + +## Upgrading to `v26.1` and later versions + +{{< include-md file="shared-content/self-managed/upgrade-notes/v26.1.md" >}} + +## Upgrading to `v26.0` + +{{< include-md file="shared-content/self-managed/upgrade-notes/v26.0.md" >}} + +## Upgrading between minor versions less than `v26` + +- Prior to `v26`, you must upgrade at most one minor version at a time. For + example, upgrading from `v25.1.5` to `v25.2.16` is permitted. diff --git a/doc/user/data/self_managed/upgrades.yml b/doc/user/data/self_managed/upgrades.yml index 8472865bd8a69..39eb0b71e11b6 100644 --- a/doc/user/data/self_managed/upgrades.yml +++ b/doc/user/data/self_managed/upgrades.yml @@ -2,8 +2,13 @@ content: | When upgrading: - - **Always** check the [version specific upgrade - notes](/self-managed-deployments/upgrading/#version-specific-upgrade-notes). + - **Always** check the [version-specific upgrade + notes](/self-managed-deployments/upgrading/version-notes/) for your target + version: + - [Upgrading to `v26.30` and later versions](/self-managed-deployments/upgrading/version-notes/#upgrading-to-v2630-and-later-versions) + - [Upgrading to `v26.1` and later versions](/self-managed-deployments/upgrading/version-notes/#upgrading-to-v261-and-later-versions) + - [Upgrading to `v26.0`](/self-managed-deployments/upgrading/version-notes/#upgrading-to-v260) + - [Upgrading between minor versions less than `v26`](/self-managed-deployments/upgrading/version-notes/#upgrading-between-minor-versions-less-than-v26) - {{< include-from-yaml data="self_managed/upgrades" name="upgrade-order-rule" >}} @@ -134,7 +139,7 @@ Select the tab for the CRD API version your deployment currently uses. This step does not cover migrating from `v1alpha1` to `v1`; to migrate, see [Adopting the v1 - CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). (The `v1` + CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). (The `v1` CRD API version is available starting in Materialize v26.30 / Terraform module v3.0.15.) @@ -189,7 +194,7 @@ Select the tab for the CRD API version your deployment currently uses. This step does not cover migrating from `v1alpha1` to `v1`; to migrate, see [Adopting the v1 - CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). (The `v1` + CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). (The `v1` CRD API version is available starting in Materialize v26.30 / Terraform module v3.0.15.) diff --git a/doc/user/layouts/partials/self-managed/crd-version-note.html b/doc/user/layouts/partials/self-managed/crd-version-note.html index b2d5023fabb42..37827002f2402 100644 --- a/doc/user/layouts/partials/self-managed/crd-version-note.html +++ b/doc/user/layouts/partials/self-managed/crd-version-note.html @@ -5,7 +5,7 @@ does for markdown links, so the raw resolves under the deployed subdirectory. */ -}} {{- $base := strings.TrimRight "/" (urls.Parse site.BaseURL).Path -}} -{{- $adopt := printf "%s/self-managed-deployments/upgrading/#adopting-the-v1-crd" $base -}} +{{- $adopt := printf "%s/self-managed-deployments/upgrading/adopting-the-v1-crd/" $base -}} {{- if eq $v "v1alpha1" -}}

                            v1alpha1 is the default CRD version. With v1alpha1, rollouts require manually rotating a UUID. For a simplified rollout behavior, see Adopting the v1 CRD.

                            {{- else -}} diff --git a/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html b/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html index c40783133131a..353098515966d 100644 --- a/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html +++ b/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html @@ -4,11 +4,11 @@ `materialize` namespace. Starting in v26.30, you can choose between CRD versions - [`v1`](/self-managed-deployments/upgrading/#upgrading-to-v2630-and-later-versions) + [`v1`](/self-managed-deployments/upgrading/version-notes/#upgrading-to-v2630-and-later-versions) and `v1alpha1`. The following enables the `v1` CRD version by setting `operator.args.installV1CRD=true`. The `v1` CRD provides a [simplified rollout - behavior](/self-managed-deployments/upgrading/#upgrading-to-v2630-and-later-versions), + behavior](/self-managed-deployments/upgrading/version-notes/#upgrading-to-v2630-and-later-versions), but requires `cert-manager`, installed in an [earlier step](#install-cert-manager) in this guide. diff --git a/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html b/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html index 4873060dcc76d..3074edcacf785 100644 --- a/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html +++ b/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html @@ -33,7 +33,7 @@ Select the tab for the CRD API version your deployment currently uses. This step does not cover migrating from `v1alpha1` to `v1`; to migrate, see [Adopting the v1 - CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). (The `v1` CRD + CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). (The `v1` CRD API version is available starting in Materialize v26.30.)
                            @@ -100,7 +100,7 @@ Select the tab for the CRD API version your deployment currently uses. This step does not cover migrating from `v1alpha1` to `v1`; to migrate, see [Adopting the v1 - CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). (The `v1` CRD + CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). (The `v1` CRD API version is available starting in Materialize v26.30.)
                            diff --git a/doc/user/shared-content/self-managed/general-rules-for-upgrades.md b/doc/user/shared-content/self-managed/general-rules-for-upgrades.md index 86f661ddcde23..c10d4e0b4fb0e 100644 --- a/doc/user/shared-content/self-managed/general-rules-for-upgrades.md +++ b/doc/user/shared-content/self-managed/general-rules-for-upgrades.md @@ -1,7 +1,7 @@ Whe upgrading: -- **Always** check the [version specific upgrade - notes](/self-managed-deployments/upgrading/#version-specific-upgrade-notes). +- **Always** check the [version-specific upgrade + notes](/self-managed-deployments/upgrading/version-notes/). - **Always** upgrade the operator **first** and ensure version compatibility between the operator and the Materialize instance you are upgrading to. diff --git a/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md b/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md index ad5ca0f9abd26..74869db743ab2 100644 --- a/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md +++ b/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md @@ -1,4 +1,4 @@ -- v26.30.0 introduces a new version of the Materialize CRD, `v1`. The new CRD simplifies rollouts by automatically detecting changes. This means you will no longer need to manually rotate a UUID to trigger a rollout. +v26.30.0 introduces a new version of the Materialize CRD, `v1`. The new CRD simplifies rollouts by automatically detecting changes. This means you will no longer need to manually rotate a UUID to trigger a rollout. Before, on `v1alpha1`: ```yaml @@ -31,7 +31,7 @@ spec: With the new change, the `requestRollout` field will be removed, along with all previously deprecated fields. -Adopting `v1` is **opt-in**. You may continue to apply `v1alpha1` CRs and your existing instances will behave exactly as before. We recommend you opt-in to `v1` at your convenience, as we will migrate to the `v1` behavior in the next major release. For step-by-step instructions, see [Adopting the v1 CRD](/self-managed-deployments/upgrading/#adopting-the-v1-crd). +Adopting `v1` is **opt-in**. You may continue to apply `v1alpha1` CRs and your existing instances will behave exactly as before. We recommend you opt-in to `v1` at your convenience, as we will migrate to the `v1` behavior in the next major release. For step-by-step instructions, see [Adopting the v1 CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). {{< important >}} From 605d2b8a3c7234fd737a289897e5ac6e9c2e6a66 Mon Sep 17 00:00:00 2001 From: kay-kim Date: Thu, 25 Jun 2026 21:46:43 -0400 Subject: [PATCH 6/6] reorg of v1 prereqs --- .../installation/install-on-local-kind.md | 28 +- .../upgrading/adopting-the-v1-crd.md | 255 ++++++++++++++++- .../upgrading/version-notes.md | 2 +- .../upgrade/upgrade-steps-local-kind.html | 4 +- .../self-managed/upgrade-notes/v26.30.md | 266 +----------------- 5 files changed, 283 insertions(+), 272 deletions(-) diff --git a/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md b/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md index 9d34874c3c8d6..3e23a1b73eed2 100644 --- a/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md +++ b/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md @@ -279,6 +279,26 @@ Starting in v26.0, Self-Managed Materialize requires a license key. 1. Install Materialize into a new `materialize-environment` namespace: + 1. If you enabled the `v1` CRD when installing the operator, update + `sample-materialize.yaml` to use it. + [`v1`](/self-managed-deployments/upgrading/adopting-the-v1-crd/) provides + a simplified rollout behavior. In the `Materialize` resource section of + the file (`kind: Materialize`): + - Change the `apiVersion` from `materialize.cloud/v1alpha1` to + `materialize.cloud/v1`. + - Remove the `requestRollout` field, if present. + + ```none + --- + apiVersion: materialize.cloud/v1 # <-- updated + kind: Materialize + metadata: + + ``` + + To use the default `v1alpha1` CRD version instead, leave + `sample-materialize.yaml` unchanged. + 1. Use the `sample-materialize.yaml` file to create the `materialize-environment` namespace and install Materialize: @@ -286,14 +306,6 @@ Starting in v26.0, Self-Managed Materialize requires a license key. kubectl apply -f sample-materialize.yaml ``` - {{< tip >}} - **Optional — use the `v1` CRD.** If you enabled the `v1` CRD on the - operator, edit `sample-materialize.yaml` to set `apiVersion: - materialize.cloud/v1` and remove any `requestRollout` field before - applying. See [Adopting the v1 - CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). - {{< /tip >}} - 1. Verify the installation and check the status: {{< note >}} diff --git a/doc/user/content/self-managed-deployments/upgrading/adopting-the-v1-crd.md b/doc/user/content/self-managed-deployments/upgrading/adopting-the-v1-crd.md index 4e8be4583ee22..0c511f0af33ef 100644 --- a/doc/user/content/self-managed-deployments/upgrading/adopting-the-v1-crd.md +++ b/doc/user/content/self-managed-deployments/upgrading/adopting-the-v1-crd.md @@ -32,14 +32,253 @@ to migrate on your own schedule before the upgrade is mandatory. ## Prerequisites -Before you begin, ensure you have completed the prerequisites in the [v26.30 -upgrade -notes](/self-managed-deployments/upgrading/version-notes/#upgrading-to-v2630-and-later-versions) -(cert-manager, network/firewall changes), and have enabled the v1 CRD by setting -the Helm value `operator.args.installV1CRD=true` on the operator. Enabling the -v1 CRD alone does not roll out your existing instances. Without this value, the -operator only installs the v1alpha1 CRD version, and the Kubernetes API server -rejects v1 CRs. +- First, install `cert-manager` (or provide your own certificate) and allow + internal network ingress on port `8001`. + +- Then enable the `v1` CRD by setting the Helm value + `operator.args.installV1CRD=true`. + +Materialize uses conversion webhooks to allow you to gracefully migrate from +`v1alpha1` to `v1`. Enabling the `v1` CRD alone does not roll out your existing +instances; they continue to use `v1alpha1`. Without the `v1` CRD, the operator +installs only the `v1alpha1` CRD version and the Kubernetes API server rejects +`v1` CRs. + +Choose the tab that matches your deployment method: + +
                            +
                              +
                              +
                              + +If you are using the [supported Terraform +modules](https://github.com/MaterializeInc/materialize-terraform-self-managed), +the required infrastructure changes (cert-manager and network ingress) will be +handled for you automatically. + +Update each module's `source` to point to the new release tag (v3.0.15 or +greater), then run `terraform init && terraform plan && terraform apply`. To +enable the `v1` CRD, set the Helm value `operator.args.installV1CRD=true` in the +values passed to the operator module. + +The key modules and their dependency chain are shown in the tabs below. Your +configuration may include additional modules (networking, storage, database, +node pools, etc.); update those to the same release tag as well. + +
                              +
                                +
                                +
                                + +```hcl +module "eks" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//aws/modules/eks?ref=" + # ... your existing configuration ... +} + +module "cert_manager" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/cert-manager?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.eks] +} + +module "operator" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//aws/modules/operator?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.cert_manager] +} + +module "materialize_instance" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/materialize-instance?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.operator] +} +``` + +For a complete example, see +[`aws/examples/simple/main.tf`](https://github.com/MaterializeInc/materialize-terraform-self-managed/blob/main/aws/examples/simple/main.tf). + +
                                +
                                + +```hcl +module "gke" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//gcp/modules/gke?ref=" + # ... your existing configuration ... +} + +module "cert_manager" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/cert-manager?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.gke] +} + +module "operator" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//gcp/modules/operator?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.cert_manager] +} + +module "materialize_instance" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/materialize-instance?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.operator] +} +``` + +For a complete example, see +[`gcp/examples/simple/main.tf`](https://github.com/MaterializeInc/materialize-terraform-self-managed/blob/main/gcp/examples/simple/main.tf). + +
                                +
                                + +```hcl +module "aks" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//azure/modules/aks?ref=" + # ... your existing configuration ... +} + +module "cert_manager" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/cert-manager?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.aks] +} + +module "operator" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//azure/modules/operator?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.cert_manager] +} + +module "materialize_instance" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/materialize-instance?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.operator] +} +``` + +For a complete example, see +[`azure/examples/simple/main.tf`](https://github.com/MaterializeInc/materialize-terraform-self-managed/blob/main/azure/examples/simple/main.tf). + +
                                +
                                +
                                +
                                +
                                + +If you are using the legacy Terraform modules +([AWS](https://github.com/MaterializeInc/terraform-aws-materialize), +[GCP](https://github.com/MaterializeInc/terraform-gcp-materialize), or +[Azure](https://github.com/MaterializeInc/terraform-azure-materialize)), +we recommend migrating to the [new supported Terraform +modules](https://github.com/MaterializeInc/materialize-terraform-self-managed) +before opting in to the `v1` CRD. + +The new modules include built-in support for the conversion webhooks used by +the `v1` CRD, including cert-manager installation and network policy +configuration. The legacy modules do not include these changes, so you would +need to apply them manually (see the **Manual** tab). + +For migration guidance, see the documentation for your cloud provider: + +- [AWS migration guide](https://github.com/MaterializeInc/materialize-terraform-self-managed/tree/main/aws/examples/migration) +- [GCP migration guide](https://github.com/MaterializeInc/materialize-terraform-self-managed/tree/main/gcp/examples/migration) +- [Azure migration guide](https://github.com/MaterializeInc/materialize-terraform-self-managed/tree/main/azure/examples/migration) + +
                                +
                                + +If you are not using our Terraform modules, you **must** complete the following +steps before enabling the `v1` CRD: + +**1. Install cert-manager** + +The conversion webhook requires a TLS certificate. +The Helm chart defaults to using [cert-manager](https://cert-manager.io/) +to automatically create and manage this certificate. cert-manager must be +installed **before** enabling the `v1` CRD. + +If you prefer to provide your own certificate instead of using cert-manager, +set the following Helm values: +- `operator.certificate.source`: `secret` +- `operator.certificate.secretName`: the name of the Kubernetes Secret + containing `ca.crt`, `tls.crt`, and `tls.key` entries. + +**2. Allow network access to the webhook port** + +The conversion webhooks require the Kubernetes API server to reach the +`orchestratord` pod on port `8001`. If your cluster enforces network policies +or cloud-level firewall rules, you must allow ingress traffic on TCP port +`8001` from the API server to pods with the label +`app.kubernetes.io/name: materialize-operator`. + +**Kubernetes NetworkPolicy:** Add a policy that allows ingress from the +Kubernetes API server on port `8001` to the `materialize-operator` pods in the +namespace where the operator is deployed: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-api-server-ingress-to-conversion-webhook + namespace: materialize # the namespace where the operator runs +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: materialize-operator + policyTypes: + - Ingress + ingress: + - ports: + - protocol: TCP + port: 8001 +``` + +**Cloud firewall rules (e.g., AWS security groups, GCP firewall rules):** +Ensure the node security group or firewall allows inbound TCP traffic on +port `8001` from the Kubernetes control plane. For example, on AWS, add an +ingress rule to the EKS node security group allowing port `8001` from the +cluster security group. On GCP with private clusters, add a firewall rule +allowing port `8001` from the GKE control plane CIDR. + +For a complete example of the required changes across AWS, Azure, and GCP, +see [this pull request](https://github.com/MaterializeInc/materialize-terraform-self-managed/pull/160). + +**3. Enable the v1 CRD** + +Once the prerequisites above are in place, set the following Helm value when +installing or upgrading the operator: + +```yaml +operator: + args: + installV1CRD: true +``` + +This installs the `v1` version of the Materialize CRD and the conversion +webhook that converts between `v1` and `v1alpha1`. + +
                                +
                                +
                                ## How it works diff --git a/doc/user/content/self-managed-deployments/upgrading/version-notes.md b/doc/user/content/self-managed-deployments/upgrading/version-notes.md index e9005df8b8bbb..5cd4735c58594 100644 --- a/doc/user/content/self-managed-deployments/upgrading/version-notes.md +++ b/doc/user/content/self-managed-deployments/upgrading/version-notes.md @@ -4,7 +4,7 @@ description: "Version-specific notes for upgrading Self-Managed Materialize." menu: main: parent: "upgrading" - weight: 5 + weight: 55 --- Review the notes for your target version before upgrading. For the general diff --git a/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html b/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html index 3074edcacf785..085dd74db25ab 100644 --- a/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html +++ b/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html @@ -108,7 +108,7 @@
                                - {{ partial "self-managed/crd-version-note.html" "v1alpha1" }} + If using `v1alpha1`, update the following fields: | Field | Description | |-------|-------------| @@ -133,6 +133,8 @@
                                + If using `v1`, update the following fields: + | Field | Description | |-------|-------------| | `environmentdImageRef` | Update the version to the new version. This should be the same as the operator version: `{{ $operator_version }}`. Updating this field automatically triggers a rollout. | diff --git a/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md b/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md index 74869db743ab2..c13fde5f471bc 100644 --- a/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md +++ b/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md @@ -31,257 +31,15 @@ spec: With the new change, the `requestRollout` field will be removed, along with all previously deprecated fields. -Adopting `v1` is **opt-in**. You may continue to apply `v1alpha1` CRs and your existing instances will behave exactly as before. We recommend you opt-in to `v1` at your convenience, as we will migrate to the `v1` behavior in the next major release. For step-by-step instructions, see [Adopting the v1 CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). - - -{{< important >}} - -Using the `v1` CRD requires infrastructure changes and an additional Helm -value, `operator.args.installV1CRD=true`. If you continue to use `v1alpha1`, -no infrastructure changes are required, but we recommend making them anyway, -as TLS certificates will become required in a future version of Materialize. - -{{< /important >}} - -Materialize uses conversion webhooks to allow you to gracefully migrate from -`v1alpha1` to `v1`. The `v1` CRD and its conversion webhook are only installed -when you set the Helm value `operator.args.installV1CRD=true`. Before enabling -it, you need to install cert-manager (or provide your own certificate) and -allow internal network ingress on port `8001`. - -Choose the tab that matches your deployment method: - -
                                -
                                  -
                                  -
                                  - -If you are using the [supported Terraform -modules](https://github.com/MaterializeInc/materialize-terraform-self-managed), -the required infrastructure changes (cert-manager and network ingress) will be -handled for you automatically. - -To upgrade, update each module's `source` to point to the new release tag and -run `terraform init && terraform plan && terraform apply`. To enable -the `v1` CRD, also set the Helm value `operator.args.installV1CRD=true` in the -values passed to the operator module. - -The key modules and their dependency chain are shown below. Your configuration -may include additional modules (networking, storage, database, node pools, etc.) -— update those to the same release tag as well. - -
                                  -
                                    -
                                    -
                                    - -```hcl -module "eks" { - source = "github.com/MaterializeInc/materialize-terraform-self-managed//aws/modules/eks?ref=" - # ... your existing configuration ... -} - -module "cert_manager" { - source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/cert-manager?ref=" - # ... your existing configuration ... - - # Your configuration may have additional dependencies here. - depends_on = [module.eks] -} - -module "operator" { - source = "github.com/MaterializeInc/materialize-terraform-self-managed//aws/modules/operator?ref=" - # ... your existing configuration ... - - # Your configuration may have additional dependencies here. - depends_on = [module.cert_manager] -} - -module "materialize_instance" { - source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/materialize-instance?ref=" - # ... your existing configuration ... - - # Your configuration may have additional dependencies here. - depends_on = [module.operator] -} -``` - -For a complete example, see -[`aws/examples/simple/main.tf`](https://github.com/MaterializeInc/materialize-terraform-self-managed/blob/main/aws/examples/simple/main.tf). - -
                                    -
                                    - -```hcl -module "gke" { - source = "github.com/MaterializeInc/materialize-terraform-self-managed//gcp/modules/gke?ref=" - # ... your existing configuration ... -} - -module "cert_manager" { - source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/cert-manager?ref=" - # ... your existing configuration ... - - # Your configuration may have additional dependencies here. - depends_on = [module.gke] -} - -module "operator" { - source = "github.com/MaterializeInc/materialize-terraform-self-managed//gcp/modules/operator?ref=" - # ... your existing configuration ... - - # Your configuration may have additional dependencies here. - depends_on = [module.cert_manager] -} - -module "materialize_instance" { - source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/materialize-instance?ref=" - # ... your existing configuration ... - - # Your configuration may have additional dependencies here. - depends_on = [module.operator] -} -``` - -For a complete example, see -[`gcp/examples/simple/main.tf`](https://github.com/MaterializeInc/materialize-terraform-self-managed/blob/main/gcp/examples/simple/main.tf). - -
                                    -
                                    - -```hcl -module "aks" { - source = "github.com/MaterializeInc/materialize-terraform-self-managed//azure/modules/aks?ref=" - # ... your existing configuration ... -} - -module "cert_manager" { - source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/cert-manager?ref=" - # ... your existing configuration ... - - # Your configuration may have additional dependencies here. - depends_on = [module.aks] -} - -module "operator" { - source = "github.com/MaterializeInc/materialize-terraform-self-managed//azure/modules/operator?ref=" - # ... your existing configuration ... - - # Your configuration may have additional dependencies here. - depends_on = [module.cert_manager] -} - -module "materialize_instance" { - source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/materialize-instance?ref=" - # ... your existing configuration ... - - # Your configuration may have additional dependencies here. - depends_on = [module.operator] -} -``` - -For a complete example, see -[`azure/examples/simple/main.tf`](https://github.com/MaterializeInc/materialize-terraform-self-managed/blob/main/azure/examples/simple/main.tf). - -
                                    -
                                    -
                                    - -
                                    -
                                    - -If you are using the legacy Terraform modules -([AWS](https://github.com/MaterializeInc/terraform-aws-materialize), -[GCP](https://github.com/MaterializeInc/terraform-gcp-materialize), or -[Azure](https://github.com/MaterializeInc/terraform-azure-materialize)), -we recommend migrating to the [new supported Terraform -modules](https://github.com/MaterializeInc/materialize-terraform-self-managed) -before opting in to the `v1` CRD. - -The new modules include built-in support for the conversion webhooks used by -the `v1` CRD, including cert-manager installation and network policy -configuration. The legacy modules do not include these changes, so you would -need to apply them manually (see the **Manual** tab). - -For migration guidance, see the documentation for your cloud provider: - -- [AWS migration guide](https://github.com/MaterializeInc/materialize-terraform-self-managed/tree/main/aws/examples/migration) -- [GCP migration guide](https://github.com/MaterializeInc/materialize-terraform-self-managed/tree/main/gcp/examples/migration) -- [Azure migration guide](https://github.com/MaterializeInc/materialize-terraform-self-managed/tree/main/azure/examples/migration) - -
                                    -
                                    - -If you are not using our Terraform modules, you **must** complete the following -steps before enabling the `v1` CRD: - -**1. Install cert-manager** - -The conversion webhook requires a TLS certificate. -The Helm chart defaults to using [cert-manager](https://cert-manager.io/) -to automatically create and manage this certificate. cert-manager must be -installed **before** enabling the `v1` CRD. - -If you prefer to provide your own certificate instead of using cert-manager, -set the following Helm values: -- `operator.certificate.source`: `secret` -- `operator.certificate.secretName`: the name of the Kubernetes Secret - containing `ca.crt`, `tls.crt`, and `tls.key` entries. - -**2. Allow network access to the webhook port** - -The conversion webhooks require the Kubernetes API server to reach the -`orchestratord` pod on port `8001`. If your cluster enforces network policies -or cloud-level firewall rules, you must allow ingress traffic on TCP port -`8001` from the API server to pods with the label -`app.kubernetes.io/name: materialize-operator`. - -**Kubernetes NetworkPolicy:** Add a policy that allows ingress from the -Kubernetes API server on port `8001` to the `materialize-operator` pods in the -namespace where the operator is deployed: - -```yaml -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-api-server-ingress-to-conversion-webhook - namespace: materialize # the namespace where the operator runs -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: materialize-operator - policyTypes: - - Ingress - ingress: - - ports: - - protocol: TCP - port: 8001 -``` - -**Cloud firewall rules (e.g., AWS security groups, GCP firewall rules):** -Ensure the node security group or firewall allows inbound TCP traffic on -port `8001` from the Kubernetes control plane. For example, on AWS, add an -ingress rule to the EKS node security group allowing port `8001` from the -cluster security group. On GCP with private clusters, add a firewall rule -allowing port `8001` from the GKE control plane CIDR. - -For a complete example of the required changes across AWS, Azure, and GCP, -see [this pull request](https://github.com/MaterializeInc/materialize-terraform-self-managed/pull/160). - -**3. Enable the v1 CRD** - -Once the prerequisites above are in place, set the following Helm value when -installing or upgrading the operator: - -```yaml -operator: - args: - installV1CRD: true -``` - -This installs the `v1` version of the Materialize CRD and the conversion -webhook that converts between `v1` and `v1alpha1`. - -
                                    -
                                    -
                                    +Upgrading to v26.30 does **not** require adopting the `v1` CRD; adopting `v1` is +**opt-in**. You can upgrade as usual while continuing to use `v1alpha1`; your existing instances will behave exactly as before. + +Adopting `v1` requires infrastructure changes (cert-manager or your own +certificate, and API-server ingress on port `8001`) and setting the Helm value +`operator.args.installV1CRD=true`; see [Adopting the v1 CRD +Prerequisites](/self-managed-deployments/upgrading/adopting-the-v1-crd/#prerequisites). + +{{< note >}} +Upgrading to v26.30 while remaining on `v1alpha1` requires **no** infrastructure +changes. +{{< /note >}}