diff --git a/bin/bump-version b/bin/bump-version index 1b6379f4c428e..96050379d276f 100755 --- a/bin/bump-version +++ b/bin/bump-version @@ -70,9 +70,11 @@ rm -f src/{clusterd,environmentd,materialized,persist-client,testdrive,catalog-d cargo update --workspace -crd_descriptions_json=doc/user/data/self_managed/materialize_crd_descriptions.json -cargo run -p mz-cloud-resources --bin crd-writer > "${crd_descriptions_json}" -git add "${crd_descriptions_json}" +for crd_version in v1alpha1 v1; do + crd_descriptions_json="doc/user/data/self_managed/materialize_crd_descriptions_${crd_version}.json" + cargo run -p mz-cloud-resources --bin crd-writer -- "${crd_version}" > "${crd_descriptions_json}" + git add "${crd_descriptions_json}" +done bin/helm-chart-version-bump --bump-orchestratord-version "v$version" diff --git a/doc/user/content/releases/_index.md b/doc/user/content/releases/_index.md index bb9953890e8d9..8ee4c8cc52105 100644 --- a/doc/user/content/releases/_index.md +++ b/doc/user/content/releases/_index.md @@ -1422,8 +1422,8 @@ See also Upgrade Notes for release specific notes. {{< include-md file="shared-content/self-managed/upgrade-notes/v26.0.md" >}} -See also [Version specific upgrade -notes](/self-managed-deployments/upgrading/#version-specific-upgrade-notes). +See also [Version-specific upgrade +notes](/self-managed-deployments/upgrading/version-notes/). ## See also diff --git a/doc/user/content/security/self-managed/authentication.md b/doc/user/content/security/self-managed/authentication.md index 6e48e7af5c7b0..ed831ce4310cf 100644 --- a/doc/user/content/security/self-managed/authentication.md +++ b/doc/user/content/security/self-managed/authentication.md @@ -50,6 +50,11 @@ following fields: The following example Kubernetes manifest includes configuration for SASL/SCRAM-SHA-256 authentication: +{{< tabs >}} +{{< tab "v1alpha1" >}} + +{{< self-managed/crd-version-note "v1alpha1" >}} + ```hc {hl_lines="15 25"} apiVersion: v1 kind: Namespace @@ -73,11 +78,48 @@ metadata: name: 12345678-1234-1234-1234-123456789012 namespace: materialize-environment spec: - environmentdImageRef: materialize/environmentd:v26.12.1 + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} backendSecretName: materialize-backend authenticatorKind: Sasl + requestRollout: 00000000-0000-0000-0000-000000000003 # Enabling auth on an existing instance requires a rollout ``` +{{< /tab >}} +{{< tab "v1" >}} + +{{< self-managed/crd-version-note "v1" >}} + +```hc {hl_lines="15 25"} +apiVersion: v1 +kind: Namespace +metadata: + name: materialize-environment +--- +apiVersion: v1 +kind: Secret +metadata: + name: materialize-backend + namespace: materialize-environment +stringData: + metadata_backend_url: "..." + persist_backend_url: "..." + license_key: "..." + external_login_password_mz_system: "enter_mz_system_password" +--- +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} + backendSecretName: materialize-backend + authenticatorKind: Sasl +``` + +{{< /tab >}} +{{< /tabs >}} + {{% include-headless "/headless/self-managed-deployments/enabled-auth-setting-warning" %}} @@ -97,6 +139,11 @@ To configure Self-Managed Materialize for password authentication, update the fo The following example Kubernetes manifest includes configuration for password authentication: +{{< tabs >}} +{{< tab "v1alpha1" >}} + +{{< self-managed/crd-version-note "v1alpha1" >}} + ```hc {hl_lines="15 25"} apiVersion: v1 kind: Namespace @@ -120,11 +167,48 @@ metadata: name: 12345678-1234-1234-1234-123456789012 namespace: materialize-environment spec: - environmentdImageRef: materialize/environmentd:v26.12.1 + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} backendSecretName: materialize-backend authenticatorKind: Password + requestRollout: 00000000-0000-0000-0000-000000000003 # Enabling auth on an existing instance requires a rollout ``` +{{< /tab >}} +{{< tab "v1" >}} + +{{< self-managed/crd-version-note "v1" >}} + +```hc {hl_lines="15 25"} +apiVersion: v1 +kind: Namespace +metadata: + name: materialize-environment +--- +apiVersion: v1 +kind: Secret +metadata: + name: materialize-backend + namespace: materialize-environment +stringData: + metadata_backend_url: "..." + persist_backend_url: "..." + license_key: "..." + external_login_password_mz_system: "enter_mz_system_password" +--- +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} + backendSecretName: materialize-backend + authenticatorKind: Password +``` + +{{< /tab >}} +{{< /tabs >}} + {{% include-headless "/headless/self-managed-deployments/enabled-auth-setting-warning" %}} diff --git a/doc/user/content/security/self-managed/sso.md b/doc/user/content/security/self-managed/sso.md index 451c7b9ebff17..77dca56b76cf3 100644 --- a/doc/user/content/security/self-managed/sso.md +++ b/doc/user/content/security/self-managed/sso.md @@ -212,6 +212,11 @@ following fields: The following example Kubernetes manifest includes configuration for OIDC authentication: +{{< tabs >}} +{{< tab "v1alpha1" >}} + +{{< self-managed/crd-version-note "v1alpha1" >}} + ```yaml {hl_lines="6-15 26 36-38"} apiVersion: v1 kind: Namespace @@ -246,7 +251,7 @@ metadata: name: 12345678-1234-1234-1234-123456789012 namespace: materialize-environment spec: - environmentdImageRef: materialize/environmentd:v26.26.0 # Use v26.26.0 or later + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} backendSecretName: materialize-backend authenticatorKind: Oidc requestRollout: 00000000-0000-0000-0000-000000000003 # Switching to Oidc requires a rollout @@ -257,6 +262,60 @@ Apply the updated manifest to your Kubernetes cluster. See [Upgrading](/self-managed-deployments/upgrading/#rollout-configuration) for details on rollout configuration. +{{< /tab >}} +{{< tab "v1" >}} + +{{< self-managed/crd-version-note "v1" >}} + +```yaml {hl_lines="6-15 26 36-37"} +apiVersion: v1 +kind: Namespace +metadata: + name: materialize-environment +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mz-system-params + namespace: materialize-environment +data: + # Create an empty system parameter configmap for later steps + system-params.json: | + { + } +--- +apiVersion: v1 +kind: Secret +metadata: + name: materialize-backend + namespace: materialize-environment +stringData: + metadata_backend_url: "..." + persist_backend_url: "..." + license_key: "..." + external_login_password_mz_system: "enter_mz_system_password" +--- +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} + backendSecretName: materialize-backend + authenticatorKind: Oidc + systemParameterConfigmapName: mz-system-params +``` + +Apply the updated manifest to your Kubernetes cluster. With the `v1` CRD, +rollouts trigger automatically when spec fields change, so no `requestRollout` +is needed. See +[Upgrading](/self-managed-deployments/upgrading/#rollout-configuration) +for details on rollout configuration. + +{{< /tab >}} +{{< /tabs >}} + {{% include-headless "/headless/self-managed-deployments/enabled-auth-setting-warning" %}} diff --git a/doc/user/content/self-managed-deployments/_index.md b/doc/user/content/self-managed-deployments/_index.md index 8965aea956ab4..03d46e946cc50 100644 --- a/doc/user/content/self-managed-deployments/_index.md +++ b/doc/user/content/self-managed-deployments/_index.md @@ -157,6 +157,11 @@ custom resource definitions(CRDs). For a full list of fields available for the Materialize CR, see [Materialize CRD Field Descriptions](/self-managed-deployments/materialize-crd-field-descriptions/). +{{< tabs >}} +{{< tab "v1alpha1" >}} + +{{< self-managed/crd-version-note "v1alpha1" >}} + ```yaml apiVersion: materialize.cloud/v1alpha1 kind: Materialize @@ -168,11 +173,35 @@ spec: # ... additional fields omitted for brevity ``` +{{< /tab >}} +{{< tab "v1" >}} + +{{< self-managed/crd-version-note "v1" >}} + +```yaml +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} +# ... additional fields omitted for brevity +``` + +{{< /tab >}} +{{< /tabs >}} + When you first apply the Materialize custom resource, the operator automatically creates all required Kubernetes resources. #### Modifying the custom resource +{{< tabs >}} +{{< tab "v1alpha1" >}} + +{{< self-managed/crd-version-note "v1alpha1" >}} + To modify a custom resource, update the CRD with your changes, including the `requestRollout` field with a new UUID value. When you apply the CRD, the operator will roll out the changes. @@ -182,6 +211,17 @@ If you do not specify a new `requestRollout` UUID, the operator watches for updates but does not roll out the changes. {{< /note >}} +{{< /tab >}} +{{< tab "v1" >}} + +{{< self-managed/crd-version-note "v1" >}} + +To modify a custom resource, update the CRD with your changes. +When you apply the CRD, the operator will roll out the changes. + +{{< /tab >}} +{{< /tabs >}} + For a full list of fields available for the Materialize CR, see [Materialize CRD Field Descriptions](/self-managed-deployments/materialize-crd-field-descriptions/). diff --git a/doc/user/content/self-managed-deployments/configuration-system-parameters.md b/doc/user/content/self-managed-deployments/configuration-system-parameters.md index 5a1604f228f4d..1c2e5969153c7 100644 --- a/doc/user/content/self-managed-deployments/configuration-system-parameters.md +++ b/doc/user/content/self-managed-deployments/configuration-system-parameters.md @@ -67,6 +67,11 @@ kubectl apply -f system-params-configmap.yaml Reference the ConfigMap in your Materialize custom resource by setting the `systemParameterConfigmapName` field to the name of your ConfigMap: +{{< tabs >}} +{{< tab "v1alpha1" >}} + +{{< self-managed/crd-version-note "v1alpha1" >}} + ```yaml {hl_lines="9-10"} apiVersion: materialize.cloud/v1alpha1 kind: Materialize @@ -74,12 +79,32 @@ metadata: name: 12345678-1234-1234-1234-123456789012 namespace: materialize-environment spec: - environmentdImageRef: materialize/environmentd:v26.0.0 + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} backendSecretName: materialize-backend systemParameterConfigmapName: mz-system-params requestRollout: 00000000-0000-0000-0000-000000000003 # Changing the CR requires a rollout ``` +{{< /tab >}} +{{< tab "v1" >}} + +{{< self-managed/crd-version-note "v1" >}} + +```yaml {hl_lines="9"} +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} + backendSecretName: materialize-backend + systemParameterConfigmapName: mz-system-params +``` + +{{< /tab >}} +{{< /tabs >}} + Apply the updated Materialize resource: ```shell @@ -129,6 +154,11 @@ Alternatively, you can add the `configmap-reload-trigger` annotation to your Materialize custom resource YAML and update it whenever you need to force a ConfigMap reload: +{{< tabs >}} +{{< tab "v1alpha1" >}} + +{{< self-managed/crd-version-note "v1alpha1" >}} + ```yaml apiVersion: materialize.cloud/v1alpha1 kind: Materialize @@ -141,6 +171,26 @@ spec: # ... rest of spec ``` +{{< /tab >}} +{{< tab "v1" >}} + +{{< self-managed/crd-version-note "v1" >}} + +```yaml +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment + annotations: + configmap-reload-trigger: "1234567890" # Update this value to force reload +spec: + # ... rest of spec +``` + +{{< /tab >}} +{{< /tabs >}} + {{< note >}} Even after the ConfigMap is synced, some system parameters may require a restart to take effect. diff --git a/doc/user/content/self-managed-deployments/installation/install-on-aws.md b/doc/user/content/self-managed-deployments/installation/install-on-aws.md index 90e0abb408ca1..bc575027eefd3 100644 --- a/doc/user/content/self-managed-deployments/installation/install-on-aws.md +++ b/doc/user/content/self-managed-deployments/installation/install-on-aws.md @@ -156,17 +156,24 @@ authentication mechanisms. 1. Create a `terraform.tfvars` file with the following variables: - - `name_prefix`: Prefix for all resource names (e.g., `simple-demo`) - - `aws_region`: AWS region for deployment (e.g., `us-east-1`) - - `aws_profile`: AWS CLI profile to use - - `license_key`: Materialize license key - - `tags`: Map of tags to apply to resources + | Variable | Description | + | ----------- | ----------------------------| + | `name_prefix` | Prefix for all resource names (e.g., `simple-demo`). | + | `aws_region` | AWS region for deployment (e.g., `us-east-1`). | + | `aws_profile` | AWS CLI profile to use. | + | `license_key` | Materialize license key. | + | `crd_version` | CRD API version to use for the Materialize instance: `v1` (recommended) or `v1alpha1`. | + | `tags` | Map of tags to apply to resources. | + + {{% include-from-yaml data="self_managed/installation" + name="installation-tfvars-crd-version-tip" %}} ```hcl name_prefix = "simple-demo" aws_region = "us-east-1" aws_profile = "your-aws-profile" license_key = "your-materialize-license-key" + crd_version = "v1" # Recommended. Default is v1alpha1; v1 is available starting in Materialize v26.30. tags = { environment = "demo" } diff --git a/doc/user/content/self-managed-deployments/installation/install-on-azure.md b/doc/user/content/self-managed-deployments/installation/install-on-azure.md index 199c6ddb8c2fa..32bdd6bf5a3f9 100644 --- a/doc/user/content/self-managed-deployments/installation/install-on-azure.md +++ b/doc/user/content/self-managed-deployments/installation/install-on-azure.md @@ -173,13 +173,18 @@ authentication mechanisms. 1. Create a `terraform.tfvars` file with the following variables: - - `subscription_id`: Azure subscription ID - - `resource_group_name`: Name for the resource group to create (e.g. - `mz-demo-rg`) - - `name_prefix`: Prefix for all resource names (e.g., `simple-demo`) - - `location`: Azure region for deployment (e.g., `westus2`) - - `license_key`: Materialize license key - - `tags`: Map of tags to apply to resources + | Variable | Description | + | ----------- | ----------------------------| + | `subscription_id` | Azure subscription ID. | + | `resource_group_name` | Name for the resource group to create (e.g., `mz-demo-rg`). | + | `name_prefix` | Prefix for all resource names (e.g., `simple-demo`). | + | `location` | Azure region for deployment (e.g., `westus2`). | + | `license_key` | Materialize license key. | + | `crd_version` | CRD API version to use for the Materialize instance: `v1` (recommended) or `v1alpha1`. | + | `tags` | Map of tags to apply to resources. | + + {{% include-from-yaml data="self_managed/installation" + name="installation-tfvars-crd-version-tip" %}} ```hcl subscription_id = "your-subscription-id" @@ -187,6 +192,7 @@ authentication mechanisms. name_prefix = "simple-demo" location = "westus2" license_key = "your-materialize-license-key" + crd_version = "v1" # Recommended. Default is v1alpha1; v1 is available starting in Materialize v26.30. tags = { environment = "demo" } diff --git a/doc/user/content/self-managed-deployments/installation/install-on-gcp.md b/doc/user/content/self-managed-deployments/installation/install-on-gcp.md index 166d63b2c8738..0106655ba34a1 100644 --- a/doc/user/content/self-managed-deployments/installation/install-on-gcp.md +++ b/doc/user/content/self-managed-deployments/installation/install-on-gcp.md @@ -187,13 +187,18 @@ authentication mechanisms. | `name_prefix` | Set a prefix for all resource names (e.g., `simple-demo`) as well as your release name for the Operator | | `region` | Set the GCP region for the deployment (e.g., `us-central1`). | | `license_key` | Set to your Materialize license key. | + | `crd_version` | Set the CRD API version to use for the Materialize instance: `v1` (recommended) or `v1alpha1`. | | `labels` | Set to the labels to apply to resources. | - ```bash + {{% include-from-yaml data="self_managed/installation" + name="installation-tfvars-crd-version-tip" %}} + + ```hcl project_id = "my-gcp-project" name_prefix = "simple-demo" region = "us-central1" license_key = "your-materialize-license-key" + crd_version = "v1" # Recommended. Default is v1alpha1; v1 is available starting in Materialize v26.30. labels = { environment = "demo" created_by = "terraform" diff --git a/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md b/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md index 5c30e1cd52c8e..3e23a1b73eed2 100644 --- a/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md +++ b/doc/user/content/self-managed-deployments/installation/install-on-local-kind.md @@ -107,13 +107,12 @@ Starting in v26.0, Self-Managed Materialize requires a license key. kubectl get nodes --show-labels ``` -1. Recommended: Install cert-manager - - Cert-manager is used for generating TLS certificates needed by the materialize operator - for CRD conversion webhooks. It is currently only required if you enable the v1 - version of the Materialize CRD by setting `operator.args.installV1CRD=true` - when installing the operator, but certificates will become required in a - future version of Materialize. +1. Install `cert-manager`. `cert-manager` is + used for generating TLS certificates needed by the Materialize operator for + CRD conversion webhooks. It is only required if you enable the `v1` version + of the Materialize CRD when installing the operator. However, we recommend + installing it even if you are not yet enabling `v1` to simplify the future + transition. ```shell helm install cert-manager oci://quay.io/jetstack/charts/cert-manager \ @@ -280,6 +279,26 @@ Starting in v26.0, Self-Managed Materialize requires a license key. 1. Install Materialize into a new `materialize-environment` namespace: + 1. If you enabled the `v1` CRD when installing the operator, update + `sample-materialize.yaml` to use it. + [`v1`](/self-managed-deployments/upgrading/adopting-the-v1-crd/) provides + a simplified rollout behavior. In the `Materialize` resource section of + the file (`kind: Materialize`): + - Change the `apiVersion` from `materialize.cloud/v1alpha1` to + `materialize.cloud/v1`. + - Remove the `requestRollout` field, if present. + + ```none + --- + apiVersion: materialize.cloud/v1 # <-- updated + kind: Materialize + metadata: + + ``` + + To use the default `v1alpha1` CRD version instead, leave + `sample-materialize.yaml` unchanged. + 1. Use the `sample-materialize.yaml` file to create the `materialize-environment` namespace and install Materialize: diff --git a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions.md b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/_index.md similarity index 51% rename from doc/user/content/self-managed-deployments/materialize-crd-field-descriptions.md rename to doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/_index.md index bca28ab7a50d5..e140b620c7a64 100644 --- a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions.md +++ b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/_index.md @@ -1,6 +1,7 @@ --- title: "Materialize CRD Field Descriptions" description: "Reference page on Materialize CRD Fields" +disable_list: true menu: main: parent: "sm-deployments" @@ -10,4 +11,7 @@ aliases: - /installation/appendix-materialize-crd-field-descriptions/ --- -{{% self-managed/materialize-crd-descriptions %}} +Select the CRD API version for your Materialize deployment: + +- [v1 (Available starting v26.30+)](/self-managed-deployments/materialize-crd-field-descriptions/v1/) +- [v1alpha1 (*default*)](/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1/) diff --git a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md new file mode 100644 index 0000000000000..b1ddd51bf2f03 --- /dev/null +++ b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1.md @@ -0,0 +1,17 @@ +--- +title: "v1" +description: "Reference page on Materialize CRD Fields for the v1 API (v26.30+)" +menu: + main: + parent: "materialize-crd-field-descriptions" + weight: 10 +--- + +{{< note >}} +The `v1` CRD is available starting in v26.30 and is opt-in. If you are on +`v1alpha1` and want a simplified rollout behavior of `v1`, see [Adopting the v1 +CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). +For the `v1alpha1` field descriptions, see [CRD v1alpha1 field descriptions](/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1/). +{{}} + +{{% self-managed/materialize-crd-descriptions-v1 %}} diff --git a/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md new file mode 100644 index 0000000000000..a08d254cb27fb --- /dev/null +++ b/doc/user/content/self-managed-deployments/materialize-crd-field-descriptions/v1alpha1.md @@ -0,0 +1,19 @@ +--- +title: "v1alpha1" +description: "Reference page on Materialize CRD Fields for the v1alpha1 API (before v26.30)" +menu: + main: + parent: "materialize-crd-field-descriptions" + weight: 20 +--- + +{{< note >}} +`v1alpha1` is the default CRD version (default). With v1alpha1, rollouts require manually rotating a UUID. +Starting in v26.30, the [v1](/self-managed-deployments/materialize-crd-field-descriptions/v1/) +CRD is available and provides a simplified rollout behavior. + +To switch to `v1`, see [Adopting the v1 +CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). +{{}} + +{{% self-managed/materialize-crd-descriptions-v1alpha1 %}} diff --git a/doc/user/content/self-managed-deployments/upgrading/_index.md b/doc/user/content/self-managed-deployments/upgrading/_index.md index 9471a91bf58a8..fc50f137ddc2e 100644 --- a/doc/user/content/self-managed-deployments/upgrading/_index.md +++ b/doc/user/content/self-managed-deployments/upgrading/_index.md @@ -93,186 +93,151 @@ helm upgrade -n materialize my-demo materialize/operator \ ## Upgrading Materialize Instances -**After** you have upgraded your Materialize Operator, upgrade your Materialize -instance(s) to the **APP Version** of the Operator. To find the version of your -currently deployed Materialize Operator: +After upgrading the operator, upgrade each Materialize instance to the +operator's **App Version** by updating its `environmentdImageRef`. For +step-by-step instructions, use the [upgrade guide](#upgrade-guides) for your +environment (Kind, AWS, Azure, or GCP). This section explains how instance +rollouts work. -```shell -helm list -n materialize -``` - -You will use the returned **App Version** for the updated `environmentdImageRef` -value. Specifically, for your Materialize instance(s), set -`environmentdImageRef` value to use the new version: - -``` -spec: - environmentdImageRef: docker.io/materialize/environmentd: -``` +## Rollout configuration -To minimize unexpected downtime and avoid connection drops at critical -periods for your application, the upgrade process involves two steps: +Materialize supports two CRD API versions: `v1alpha1` (default) and `v1` +(available starting in v26.30). -- First, stage the changes (update the `environmentdImageRef` with the new - version) to the Materialize custom resource. The Operator watches for changes - but does not automatically roll out the changes. +How you trigger a rollout depends on the CRD API version of your instances. -- Second, roll out the changes by specifying a new UUID for `requestRollout`. +{{< tip >}} +To migrate a `v1alpha1` instance to `v1`, see +[Adopting the v1 CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). +{{< /tip >}} -### Stage the Materialize instance version change +{{< tabs >}} +{{< tab "v1alpha1" >}} -To stage the Materialize instances version upgrade, update the -`environmentdImageRef` field in the Materialize custom resource spec to the -compatible version of your currently deployed Materialize Operator. - -To stage, but **not** rollout, the Materialize instance version upgrade, you can -use the `kubectl patch` command; for example, if the **App Version** is {{< self-managed/versions/get-latest-version >}}: - -```shell -kubectl patch materialize \ - -n \ - --type='merge' \ - -p "{\"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\"}}" -``` +Specify a new `UUID` value for `requestRollout` to roll out changes to the +Materialize instance. {{< note >}} -Until you specify a new `requestRollout`, the Operator watches for updates but -does not roll out the changes. +`requestRollout` without the `forceRollout` field only rolls out if changes exist +to the Materialize instance. To roll out even if there are no changes to the +instance, use it with `forceRollout`. {{< /note >}} - -### Applying the changes via `requestRollout` - -To apply chang Materialize instance upgrade, you must update the `requestRollout` field in the Materialize custom resource spec to a new UUID. -Be sure to consult the [Rollout Configurations](#rollout-configuration) to ensure you've selected the correct rollout behavior. ```shell -# Then trigger the rollout with a new UUID +# Only rolls out if there are changes kubectl patch materialize \ -n \ --type='merge' \ -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\"}}" ``` -### Staging and applying in a single command - -Although separating the staging and rollout of the changes into two steps can -minimize unexpected downtime and avoid connection drops at critical periods, you -can, if preferred, combine both operations in a single command +To force a rollout even when there are no other changes, set a new `forceRollout` +UUID alongside `requestRollout`: ```shell kubectl patch materialize \ - -n materialize-environment \ + -n \ --type='merge' \ - -p "{\"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\", \"requestRollout\": \"$(uuidgen)\"}}" -``` - -#### Using YAML Definition - -Alternatively, you can update your Materialize custom resource definition directly: - -```yaml -apiVersion: materialize.cloud/v1alpha1 -kind: Materialize -metadata: - name: 12345678-1234-1234-1234-123456789012 - namespace: materialize-environment -spec: - environmentdImageRef: materialize/environmentd:{{< self-managed/versions/get-latest-version >}} # Update version as needed - requestRollout: 22222222-2222-2222-2222-222222222222 # Use a new UUID - forceRollout: 33333333-3333-3333-3333-333333333333 # Optional: for forced rollouts - inPlaceRollout: false # In Place rollout is deprecated and ignored. Please use rolloutStrategy - rolloutStrategy: WaitUntilReady # The mechanism to use when rolling out the new version. Can be WaitUntilReady or ImmediatelyPromoteCausingDowntime - backendSecretName: materialize-backend -``` - -Apply the updated definition: - -```shell -kubectl apply -f materialize.yaml + -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\", \"forceRollout\": \"$(uuidgen)\"}}" ``` -## Rollout Configuration +{{< /tab >}} +{{< tab "v1" >}} -### `requestRollout` +With v1, updating the spec automatically triggers a rollout and there is no +`requestRollout` field. For details on the underlying mechanism, see [How it +works](/self-managed-deployments/upgrading/adopting-the-v1-crd/#how-it-works). -Specify a new `UUID` value for the `requestRollout` to roll out the changes to -the Materialize instance. +To trigger a rollout even when there are no other changes to the instance, +specify a new `UUID` value for `forceRollout`. That is: -{{< note >}} +- Set it in your manifest: -`requestRollout` without the `forcedRollout` field only rolls out if changes -exist to the Materialize instance. To roll out even if there are no changes to -the instance, use with `forcedRollouts`. + ```yaml + spec: + forceRollout: # e.g. the output of `uuidgen` + ``` -{{< /note >}} +- Then reapply. -```shell -# Only rolls out if there are changes -kubectl patch materialize \ - -n \ - --type='merge' \ - -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\"}}" -``` -#### `requestRollout` with `forcedRollouts` - -Specify a new `UUID` value for `forcedRollout` to roll out even when there are -no changes to the instance. Use `forcedRollout` with `requestRollout`. - -```shell -kubectl patch materialize \ - -n materialize-environment \ - --type='merge' \ - -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\", \"forceRollout\": \"$(uuidgen)\"}}" -``` + ```shell + kubectl apply -f materialize.yaml + ``` -### Rollout strategies +{{< /tab >}} +{{< /tabs >}} -Rollout strategies control how Materialize transitions from the current generation to a new generation during an upgrade. +## Rollout strategies -The behavior of the new version rollout follows your `rolloutStrategy` setting. +Rollout strategies control how Materialize transitions from the current +generation to a new generation during an upgrade. The behavior follows your +`rolloutStrategy` setting. -#### *WaitUntilReady* - ***Default*** +### *WaitUntilReady* — ***Default*** -`WaitUntilReady` creates a new generation of pods and automatically promotes -them as soon as they catch up to the old generation and become `ReadyToPromote`. +`WaitUntilReady` creates a new generation of pods and automatically promotes them +as soon as they catch up to the old generation and become `ReadyToPromote`. Because both generations run simultaneously until the promotion, this strategy temporarily doubles the required resources to run Materialize. {{< note >}} - -Starting in v26.28.0, `WaitUntilReady` waits up to 1 year (configurable by the -`with_0dt_deployment_max_wait` flag) for the new pods to become -`ReadyToPromote`. If the pods do not reach `ReadyToPromote` by then, the rollout -is cancelled and the new pods are removed. - -Earlier versions used a default wait of 72 hours, and, once the wait time -expired, the new pods were automatically promoted regardless of their readiness. +While the new generation waits to become `ReadyToPromote`, it runs in a +read-only, un-promoted state and holds back compaction. To prevent it from +sitting in this state indefinitely (which can cause incident-inducing load when +it is eventually promoted), the rollout is bounded by the `rolloutRequestTimeout` +field in the Materialize spec, which defaults to `24h`. + +If the new generation does not become `ReadyToPromote` within +`rolloutRequestTimeout`, the operator cancels the rollout: the new generation is +torn down and the previously-active generation continues serving. You can then +trigger a new rollout (in v1, set a new `forceRollout`; in v1alpha1, set a new +`requestRollout`). {{< /note >}} -#### *ImmediatelyPromoteCausingDowntime* +### *ImmediatelyPromoteCausingDowntime* + {{< warning >}} Using the `ImmediatelyPromoteCausingDowntime` rollout flag will cause downtime. {{< /warning >}} -`ImmediatelyPromoteCausingDowntime` tears down the prior generation, and immediately promotes the new generation without waiting for it to hydrate. This causes downtime until the new generation has hydrated. However, it does not require additional resources. +`ImmediatelyPromoteCausingDowntime` tears down the prior generation and +immediately promotes the new generation without waiting for it to hydrate. This +causes downtime until the new generation has hydrated. However, it does not +require additional resources. -#### *ManuallyPromote* +### *ManuallyPromote* -`ManuallyPromote` allows you to choose when to promote the new generation. This means you can time the promotion for periods when load is low, minimizing the impact of potential downtime for any clients connected to Materialize. This strategy temporarily doubles the required resources to run Materialize. +`ManuallyPromote` allows you to choose when to promote the new generation. This +means you can time the promotion for periods when load is low, minimizing the +impact of potential downtime for any clients connected to Materialize. This +strategy temporarily doubles the required resources to run Materialize. -To minimize downtime, wait until the new generation has fully hydrated and caught up to the prior generation before promoting. To check hydration status, inspect the `UpToDate` condition in the Materialize resource status. When hydration completes, the condition will be `ReadyToPromote`. +To minimize downtime, wait until the new generation has fully hydrated and caught +up to the prior generation before promoting. To check hydration status, inspect +the `UpToDate` condition in the Materialize resource status. When hydration +completes, the condition will be `ReadyToPromote`. -To promote, update the `forcePromote` field to match the `requestRollout` field in the Materialize spec. If you need to promote before hydration completes, you can set `forcePromote` immediately, but clients may experience downtime. +To promote, update the `forcePromote` field to match the current rollout +identifier (in v1, the `status.requestedRolloutHash`; in v1alpha1, the +`requestRollout` UUID in the spec). If you need to promote before hydration +completes, you can set `forcePromote` immediately, but clients may experience +downtime. {{< warning >}} Leaving a new generation unpromoted for over 6 hours may cause downtime. {{< /warning >}} -**Do not leave new generations unpromoted indefinitely**. They should either be promoted or canceled. New generations open a read hold on the metadata database that prevents compaction. This hold is only released when the generation is promoted or canceled. If left open too long, promoting or canceling can trigger a spike in deletion load on the metadata database, potentially causing downtime. It is not recommended to leave generations unpromoted for over 6 hours. +**Do not leave new generations unpromoted indefinitely**. They should either be +promoted or canceled. New generations open a read hold on the metadata database +that prevents compaction. This hold is only released when the generation is +promoted or canceled. If left open too long, promoting or canceling can trigger a +spike in deletion load on the metadata database, potentially causing downtime. It +is not recommended to leave generations unpromoted for over 6 hours. -#### *inPlaceRollout* - ***Deprecated*** +### *inPlaceRollout* — ***Deprecated*** (v1alpha1 only) The setting is ignored. -## Verifying the Upgrade +## Verifying the upgrade -After initiating the rollout, you can monitor the status field of the Materialize custom resource to check on the upgrade. +After initiating the rollout, you can monitor the status field of the Materialize +custom resource to check on the upgrade. ```shell # Watch the status of your Materialize environment @@ -282,16 +247,22 @@ kubectl get materialize -n materialize-environment -w kubectl logs -l app.kubernetes.io/name=materialize-operator -n materialize ``` -## Cancelling the Upgrade +## Cancelling the upgrade + +You may want to cancel an in-progress rollout if the upgrade has failed (for +example, new pods are not healthy). Before cancelling, verify that the upgrade has +not already completed by checking that the deploy generation (found via +`status.activeGeneration`) is still the one from before the upgrade. Once an +upgrade has already happened, you cannot revert using this method. -You may want to cancel an in-progress rollout if the upgrade has failed. This may be indicated by new pods not being healthy. Before cancelling, verify that the upgrade has not already completed by checking that the deploy generation (found via `status.activeGeneration`) is still the one from before the upgrade. Once an upgrade has already happened, you cannot revert using this method. +{{< tabs >}} +{{< tab "v1alpha1" >}} -To cancel an in-progress rollout and revert to the last completed rollout -state, revert both `requestRollout` and `environmentdImageRef` back to the -values from the last completed rollout. Reverting `environmentdImageRef` -alongside `requestRollout` keeps the spec aligned with what is actually -running, so a later rollout doesn't accidentally pick up the previously -attempted upgrade image. +To cancel an in-progress rollout and revert to the last completed rollout state, +revert both `requestRollout` and `environmentdImageRef` back to the values from +the last completed rollout. Reverting `environmentdImageRef` alongside +`requestRollout` keeps the spec aligned with what is actually running, so a later +rollout doesn't accidentally pick up the previously attempted upgrade image. First, retrieve the last completed rollout request ID and the matching environmentd image ref from your Materialize CR: @@ -301,7 +272,7 @@ kubectl get materialize -n materialize-environment \ -o jsonpath='{.status.lastCompletedRolloutRequest} {.status.lastCompletedRolloutEnvironmentdImageRef}' ``` -Then, set both fields back to these values in a single patch: +Then set both fields back to these values in a single patch: ```shell kubectl patch materialize \ @@ -310,21 +281,29 @@ kubectl patch materialize \ -p "{\"spec\": {\"requestRollout\": \"\", \"environmentdImageRef\": \"\"}}" ``` -## Version Specific Upgrade Notes +{{< /tab >}} +{{< tab "v1" >}} -### Upgrading to `v26.1` and later versions +To cancel an in-progress rollout and revert to the last completed rollout state, +reapply the Materialize resource with the spec it had before the rollout (notably +the previous `environmentdImageRef`). Because v1 derives the rollout from the +spec hash, restoring the previous spec produces the previous hash and returns the +instance to the last completed state. -{{< include-md file="shared-content/self-managed/upgrade-notes/v26.1.md" >}} +```shell +kubectl apply -f previous_materialize_configuration.yaml +``` -### Upgrading to `v26.0` +{{< /tab >}} +{{< /tabs >}} -{{< include-md file="shared-content/self-managed/upgrade-notes/v26.0.md" >}} +## See also -### Upgrading between minor versions less than `v26` - - Prior to `v26`, you must upgrade at most one minor version at a time. For - example, upgrading from `v25.1.5` to `v25.2.16` is permitted. +- [Version-specific upgrade + notes](/self-managed-deployments/upgrading/version-notes/) -## See also +- [Adopting the v1 + CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/) - [Materialize Operator Configuration](/self-managed-deployments/operator-configuration/) diff --git a/doc/user/content/self-managed-deployments/upgrading/adopting-the-v1-crd.md b/doc/user/content/self-managed-deployments/upgrading/adopting-the-v1-crd.md new file mode 100644 index 0000000000000..0c511f0af33ef --- /dev/null +++ b/doc/user/content/self-managed-deployments/upgrading/adopting-the-v1-crd.md @@ -0,0 +1,341 @@ +--- +title: "Adopting the v1 CRD" +description: "Adopt the v1 Materialize CRD API version for Self-Managed Materialize." +menu: + main: + parent: "upgrading" + weight: 80 +--- + +This page describes the Materialize CRD API versions and how to adopt `v1` for +your Materialize instances. + +## CRD API versions + +Starting in v26.30, the Materialize Operator supports two CRD API versions: + +- **v1alpha1** (default) uses a two-step rollout: first stage the spec change, + then trigger a rollout with a new `requestRollout` UUID. +- **v1** simplifies upgrades — rollouts trigger automatically when spec fields + change, removing the need to manually set a `requestRollout` UUID. + +Adopting v1 is **opt-in** for now. Upgrading the operator to v26.30+ does not +change your existing v1alpha1 CRs or their behavior; you can continue using +v1alpha1 and adopt v1 for individual instances at your own pace. + +{{< important >}} +In the next major release, all Materialize CRs will be force upgraded to v1. You +will still be able to apply v1alpha1 CRs, but they will be auto-converted to v1 +and use the v1 rollout behavior. We recommend opting in to v1 at your convenience +to migrate on your own schedule before the upgrade is mandatory. +{{< /important >}} + +## Prerequisites + +- First, install `cert-manager` (or provide your own certificate) and allow + internal network ingress on port `8001`. + +- Then enable the `v1` CRD by setting the Helm value + `operator.args.installV1CRD=true`. + +Materialize uses conversion webhooks to allow you to gracefully migrate from +`v1alpha1` to `v1`. Enabling the `v1` CRD alone does not roll out your existing +instances; they continue to use `v1alpha1`. Without the `v1` CRD, the operator +installs only the `v1alpha1` CRD version and the Kubernetes API server rejects +`v1` CRs. + +Choose the tab that matches your deployment method: + +
+
    +
    +
    + +If you are using the [supported Terraform +modules](https://github.com/MaterializeInc/materialize-terraform-self-managed), +the required infrastructure changes (cert-manager and network ingress) will be +handled for you automatically. + +Update each module's `source` to point to the new release tag (v3.0.15 or +greater), then run `terraform init && terraform plan && terraform apply`. To +enable the `v1` CRD, set the Helm value `operator.args.installV1CRD=true` in the +values passed to the operator module. + +The key modules and their dependency chain are shown in the tabs below. Your +configuration may include additional modules (networking, storage, database, +node pools, etc.); update those to the same release tag as well. + +
    +
      +
      +
      + +```hcl +module "eks" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//aws/modules/eks?ref=" + # ... your existing configuration ... +} + +module "cert_manager" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/cert-manager?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.eks] +} + +module "operator" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//aws/modules/operator?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.cert_manager] +} + +module "materialize_instance" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/materialize-instance?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.operator] +} +``` + +For a complete example, see +[`aws/examples/simple/main.tf`](https://github.com/MaterializeInc/materialize-terraform-self-managed/blob/main/aws/examples/simple/main.tf). + +
      +
      + +```hcl +module "gke" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//gcp/modules/gke?ref=" + # ... your existing configuration ... +} + +module "cert_manager" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/cert-manager?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.gke] +} + +module "operator" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//gcp/modules/operator?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.cert_manager] +} + +module "materialize_instance" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/materialize-instance?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.operator] +} +``` + +For a complete example, see +[`gcp/examples/simple/main.tf`](https://github.com/MaterializeInc/materialize-terraform-self-managed/blob/main/gcp/examples/simple/main.tf). + +
      +
      + +```hcl +module "aks" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//azure/modules/aks?ref=" + # ... your existing configuration ... +} + +module "cert_manager" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/cert-manager?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.aks] +} + +module "operator" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//azure/modules/operator?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.cert_manager] +} + +module "materialize_instance" { + source = "github.com/MaterializeInc/materialize-terraform-self-managed//kubernetes/modules/materialize-instance?ref=" + # ... your existing configuration ... + + # Your configuration may have additional dependencies here. + depends_on = [module.operator] +} +``` + +For a complete example, see +[`azure/examples/simple/main.tf`](https://github.com/MaterializeInc/materialize-terraform-self-managed/blob/main/azure/examples/simple/main.tf). + +
      +
      +
      +
      +
      + +If you are using the legacy Terraform modules +([AWS](https://github.com/MaterializeInc/terraform-aws-materialize), +[GCP](https://github.com/MaterializeInc/terraform-gcp-materialize), or +[Azure](https://github.com/MaterializeInc/terraform-azure-materialize)), +we recommend migrating to the [new supported Terraform +modules](https://github.com/MaterializeInc/materialize-terraform-self-managed) +before opting in to the `v1` CRD. + +The new modules include built-in support for the conversion webhooks used by +the `v1` CRD, including cert-manager installation and network policy +configuration. The legacy modules do not include these changes, so you would +need to apply them manually (see the **Manual** tab). + +For migration guidance, see the documentation for your cloud provider: + +- [AWS migration guide](https://github.com/MaterializeInc/materialize-terraform-self-managed/tree/main/aws/examples/migration) +- [GCP migration guide](https://github.com/MaterializeInc/materialize-terraform-self-managed/tree/main/gcp/examples/migration) +- [Azure migration guide](https://github.com/MaterializeInc/materialize-terraform-self-managed/tree/main/azure/examples/migration) + +
      +
      + +If you are not using our Terraform modules, you **must** complete the following +steps before enabling the `v1` CRD: + +**1. Install cert-manager** + +The conversion webhook requires a TLS certificate. +The Helm chart defaults to using [cert-manager](https://cert-manager.io/) +to automatically create and manage this certificate. cert-manager must be +installed **before** enabling the `v1` CRD. + +If you prefer to provide your own certificate instead of using cert-manager, +set the following Helm values: +- `operator.certificate.source`: `secret` +- `operator.certificate.secretName`: the name of the Kubernetes Secret + containing `ca.crt`, `tls.crt`, and `tls.key` entries. + +**2. Allow network access to the webhook port** + +The conversion webhooks require the Kubernetes API server to reach the +`orchestratord` pod on port `8001`. If your cluster enforces network policies +or cloud-level firewall rules, you must allow ingress traffic on TCP port +`8001` from the API server to pods with the label +`app.kubernetes.io/name: materialize-operator`. + +**Kubernetes NetworkPolicy:** Add a policy that allows ingress from the +Kubernetes API server on port `8001` to the `materialize-operator` pods in the +namespace where the operator is deployed: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-api-server-ingress-to-conversion-webhook + namespace: materialize # the namespace where the operator runs +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: materialize-operator + policyTypes: + - Ingress + ingress: + - ports: + - protocol: TCP + port: 8001 +``` + +**Cloud firewall rules (e.g., AWS security groups, GCP firewall rules):** +Ensure the node security group or firewall allows inbound TCP traffic on +port `8001` from the Kubernetes control plane. For example, on AWS, add an +ingress rule to the EKS node security group allowing port `8001` from the +cluster security group. On GCP with private clusters, add a firewall rule +allowing port `8001` from the GKE control plane CIDR. + +For a complete example of the required changes across AWS, Azure, and GCP, +see [this pull request](https://github.com/MaterializeInc/materialize-terraform-self-managed/pull/160). + +**3. Enable the v1 CRD** + +Once the prerequisites above are in place, set the following Helm value when +installing or upgrading the operator: + +```yaml +operator: + args: + installV1CRD: true +``` + +This installs the `v1` version of the Materialize CRD and the conversion +webhook that converts between `v1` and `v1alpha1`. + +
      +
      +
      + +## How it works + +The v1alpha1 CRD remains the storage version. When you submit a v1 CR, the +operator's conversion webhook automatically converts it to v1alpha1 for storage. +During conversion, the webhook computes a SHA256 hash of a subset of the spec +fields and derives a deterministic `requestRollout` UUID from it. The hash covers +the fields that affect the running `environmentd` (for example, +`environmentdImageRef`, `environmentdExtraArgs`, `environmentdExtraEnv`, resource +requirements, `podAnnotations`, `podLabels`, `authenticatorKind`, `enableRbac`, +`rolloutStrategy`, and `forceRollout`). It excludes fields that do not require a +rollout, such as `balancerd`/`console` resource requirements and replica counts. +The operator triggers a rollout whenever `requestRollout` differs from the last +completed rollout, so: + +- **Adopting v1 on an existing v1alpha1 instance typically triggers one + rollout.** The derived `requestRollout` is computed from the spec hash and will + not match the `requestRollout` you previously set by hand, so the instance + rolls out once even if nothing else in the spec changed. Adopt v1 during a + window where a rollout is acceptable. +- **Once on v1, an unchanged spec does not trigger a rollout.** Reapplying the + same spec produces the same hash and the same derived `requestRollout`. + Changing a hashed spec field produces a new value and triggers a rollout + automatically. + +## Using kubectl + +To adopt v1 for an existing instance, apply your CR with `apiVersion: +materialize.cloud/v1` and remove the `requestRollout` field: + +```shell +kubectl apply -f - < + namespace: +spec: + environmentdImageRef: + backendSecretName: + # ... other spec fields (copy from your existing CR, removing requestRollout) +EOF +``` + +## Using Terraform + +If you are managing your Materialize instance with the [Materialize Terraform +modules](https://github.com/MaterializeInc/materialize-terraform-self-managed), +set: + +```hcl +crd_version = "v1" +request_rollout = null +``` + +## Returning to the v1alpha1 behavior + +You can go back to the v1alpha1 rollout behavior at any time by applying your CR +with `apiVersion: materialize.cloud/v1alpha1` and an explicit `requestRollout` +UUID. diff --git a/doc/user/content/self-managed-deployments/upgrading/upgrade-on-azure.md b/doc/user/content/self-managed-deployments/upgrading/upgrade-on-azure.md index 291826d42483e..e406839107a14 100644 --- a/doc/user/content/self-managed-deployments/upgrading/upgrade-on-azure.md +++ b/doc/user/content/self-managed-deployments/upgrading/upgrade-on-azure.md @@ -66,7 +66,7 @@ The following procedure performs a rolling upgrade, where both the old and new M - `` with your cluster name; i.e., the `aks_cluster_name` in the Terraform output. For the sample example, - your cluster name has the form `{prefix_name}-aks`; e.g., simple-demo-aks`. + your cluster name has the form `{prefix_name}-aks`; e.g., `simple-demo-aks`. ```bash # az aks get-credentials --resource-group --name diff --git a/doc/user/content/self-managed-deployments/upgrading/upgrade-on-kind.md b/doc/user/content/self-managed-deployments/upgrading/upgrade-on-kind.md index 866e8a933b2b6..9e169fd470b32 100644 --- a/doc/user/content/self-managed-deployments/upgrading/upgrade-on-kind.md +++ b/doc/user/content/self-managed-deployments/upgrading/upgrade-on-kind.md @@ -29,7 +29,7 @@ If you don't have Helm version 3.2.0+ installed, install. For details, see the ### `kubectl` This tutorial uses `kubectl`. To install, refer to the [`kubectl` -documentationq](https://kubernetes.io/docs/tasks/tools/). +documentation](https://kubernetes.io/docs/tasks/tools/). For help with `kubectl` commands, see [kubectl Quick reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). @@ -43,7 +43,7 @@ reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). {{< important >}} The following procedure performs a rolling upgrade, where both the old and new -Materialize instances are running before the the old instance are removed. +Materialize instances are running before the old instances are removed. When performing a rolling upgrade, ensure you have enough resources to support having both the old and new Materialize instances running. diff --git a/doc/user/content/self-managed-deployments/upgrading/version-notes.md b/doc/user/content/self-managed-deployments/upgrading/version-notes.md new file mode 100644 index 0000000000000..5cd4735c58594 --- /dev/null +++ b/doc/user/content/self-managed-deployments/upgrading/version-notes.md @@ -0,0 +1,29 @@ +--- +title: "Version-specific upgrade notes" +description: "Version-specific notes for upgrading Self-Managed Materialize." +menu: + main: + parent: "upgrading" + weight: 55 +--- + +Review the notes for your target version before upgrading. For the general +upgrade procedure, see the [upgrade +guides](/self-managed-deployments/upgrading/#upgrade-guides). + +## Upgrading to `v26.30` and later versions + +{{< include-md file="shared-content/self-managed/upgrade-notes/v26.30.md" >}} + +## Upgrading to `v26.1` and later versions + +{{< include-md file="shared-content/self-managed/upgrade-notes/v26.1.md" >}} + +## Upgrading to `v26.0` + +{{< include-md file="shared-content/self-managed/upgrade-notes/v26.0.md" >}} + +## Upgrading between minor versions less than `v26` + +- Prior to `v26`, you must upgrade at most one minor version at a time. For + example, upgrading from `v25.1.5` to `v25.2.16` is permitted. diff --git a/doc/user/data/self_managed/installation.yml b/doc/user/data/self_managed/installation.yml index 8c6bb711d774d..390e5fe453249 100644 --- a/doc/user/data/self_managed/installation.yml +++ b/doc/user/data/self_managed/installation.yml @@ -86,9 +86,18 @@ If you run into an error during deployment, refer to the [Troubleshooting](/self-managed-deployments/troubleshooting/). +- name: installation-tfvars-crd-version-tip + content: | + {{< tip >}} + CRD API version `v1` is available starting in Materialize v26.30 and + Materialize Terraform module version v3.0.15. + For earlier versions, or if you want to remain on the default `v1alpha1`, + comment out the `crd_version` line or set it to `v1alpha1`. + {{< /tip >}} + - name: installation-tfvars-variables-optional content: | - **Optional variables**: + **Additional variables**: - `internal_load_balancer`: Flag that determines whether the load balancer is internal (default) or public. diff --git a/doc/user/data/self_managed/materialize_crd_descriptions_v1.json b/doc/user/data/self_managed/materialize_crd_descriptions_v1.json new file mode 100644 index 0000000000000..c410f849e1765 --- /dev/null +++ b/doc/user/data/self_managed/materialize_crd_descriptions_v1.json @@ -0,0 +1,558 @@ +[ + [ + "MaterializeSpec", + [ + { + "name": "backendSecretName", + "type": "String", + "description": "The name of a secret containing `metadata_backend_url` and `persist_backend_url`.\nIt may also contain `external_login_password_mz_system`, which will be used as\nthe password for the `mz_system` user if `authenticatorKind` is `Password`.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "environmentdImageRef", + "type": "String", + "description": "The environmentd image to run.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "authenticatorKind", + "type": "Enum", + "description": "How to authenticate with Materialize.\n\nValid values:\n- `Frontegg`:
      Authenticate users using Frontegg.\n- `Password`:
      Authenticate users using internally stored password hashes.\n The backend secret must contain external_login_password_mz_system.\n- `Sasl`:
      Authenticate users using SASL.\n- `Oidc`:
      Authenticate users using OIDC (JWT tokens).\n- `None` (default):
      Do not authenticate users. Trust they are who they say they are without verification.", + "default": "None", + "required": false, + "deprecated": false + }, + { + "name": "balancerdExternalCertificateSpec", + "type": "MaterializeCertSpec", + "description": "The configuration for generating an x509 certificate using cert-manager for balancerd\nto present to incoming connections.\nThe `dnsNames` and `issuerRef` fields are required.\n\nThis field is excluded from the rollout hash and changes will not trigger a rollout.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "balancerdReplicas", + "type": "Integer", + "description": "Number of balancerd pods to create.\n\nThis field is excluded from the rollout hash and changes will not trigger a rollout.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "balancerdResourceRequirements", + "type": "io.k8s.api.core.v1.ResourceRequirements", + "description": "Resource requirements for the balancerd pod.\n\nThis field is excluded from the rollout hash and changes will not trigger a rollout.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "consoleExternalCertificateSpec", + "type": "MaterializeCertSpec", + "description": "The configuration for generating an x509 certificate using cert-manager for the console\nto present to incoming connections.\nThe `dnsNames` and `issuerRef` fields are required.\nNot yet implemented.\n\nThis field is excluded from the rollout hash and changes will not trigger a rollout.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "consoleReplicas", + "type": "Integer", + "description": "Number of console pods to create.\n\nThis field is excluded from the rollout hash and changes will not trigger a rollout.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "consoleResourceRequirements", + "type": "io.k8s.api.core.v1.ResourceRequirements", + "description": "Resource requirements for the console pod.\n\nThis field is excluded from the rollout hash and changes will not trigger a rollout.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "enableRbac", + "type": "Bool", + "description": "Whether to enable role based access control. Defaults to false.", + "default": false, + "required": false, + "deprecated": false + }, + { + "name": "environmentId", + "type": "Uuid", + "description": "The value used by environmentd (via the --environment-id flag) to\nuniquely identify this instance. Must be globally unique, and\nis required if a license key is not provided.\nNOTE: This value MUST NOT be changed in an existing instance,\nsince it affects things like the way data is stored in the persist\nbackend.", + "default": "00000000-0000-0000-0000-000000000000", + "required": false, + "deprecated": false + }, + { + "name": "environmentdConnectionRoleArn", + "type": "String", + "description": "If running in AWS, override the IAM role to use to support\nthe CREATE CONNECTION feature.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "environmentdExtraArgs", + "type": "Array", + "description": "Extra args to pass to the environmentd binary.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "environmentdExtraEnv", + "type": "Array", + "description": "Extra environment variables to pass to the environmentd binary.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "environmentdResourceRequirements", + "type": "io.k8s.api.core.v1.ResourceRequirements", + "description": "Resource requirements for the environmentd pod.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "environmentdScratchVolumeStorageRequirement", + "type": "io.k8s.apimachinery.pkg.api.resource.Quantity", + "description": "Amount of disk to allocate, if a storage class is provided.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "forcePromote", + "type": "String", + "description": "If `forcePromote` is set to the same value as the `status.requestedRolloutHash`,\ncurrent rollout will skip waiting for clusters in the new\ngeneration to rehydrate before promoting the new environmentd to\nleader.\n\nThis field is excluded from the rollout hash and changes will not trigger a rollout.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "forceRollout", + "type": "Uuid", + "description": "This value will force the controller to detect the spec as changed\neven if no other changes happened. This can be used to force a rollout\nto a new generation even without making any meaningful changes.", + "default": "00000000-0000-0000-0000-000000000000", + "required": false, + "deprecated": false + }, + { + "name": "internalCertificateSpec", + "type": "MaterializeCertSpec", + "description": "The cert-manager Issuer or ClusterIssuer to use for database internal communication.\nThe `issuerRef` field is required.\nThis currently is only used for environmentd, but will eventually support clusterd.\nNot yet implemented.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "podAnnotations", + "type": "Map", + "description": "Annotations to apply to the pods.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "podLabels", + "type": "Map", + "description": "Labels to apply to the pods.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "rolloutRequestTimeout", + "type": "RolloutRequestTimeout", + "description": "The maximum amount of time a rollout may remain in progress before\nit is automatically cancelled.\n\nWhile a rollout is in progress, the new generation of `environmentd`\nruns in a read-only, un-promoted state and holds back compaction via\nread holds. Leaving it in this state for too long can cause\nincident-inducing load when it is eventually promoted, so the\noperator cancels the rollout once this timeout is exceeded: the new\ngeneration is torn down and the previously-active generation\ncontinues serving. A new rollout can then be triggered by setting\n`forceRollout` to a new value.\n\nThis does not apply to the `ImmediatelyPromoteCausingDowntime`\nrollout strategy or to force-promoted rollouts, since by the time\nthose are in progress the old generation may already be gone.\n\nThe value is parsed as a human-readable duration, e.g. `24h`,\n`90m`, or `1h 30m`. Defaults to [`DEFAULT_ROLLOUT_REQUEST_TIMEOUT`]\nwhen omitted (the API server fills it in); an unparseable value also\nfalls back to that default.", + "default": "24h", + "required": false, + "deprecated": false + }, + { + "name": "rolloutStrategy", + "type": "Enum", + "description": "Rollout strategy to use when upgrading this Materialize instance.\n\nValid values:\n- `WaitUntilReady` (default):
      Create a new generation of pods, leaving the old generation around until the\n new ones are ready to take over.\n This minimizes downtime, and is what almost everyone should use.\n- `ManuallyPromote`:
      Create a new generation of pods, leaving the old generation as the serving generation\n until the user manually promotes the new generation.\n \n When using `ManuallyPromote`, the new generation can be promoted at any\n time, even if it has dataflows that are not fully caught up, by setting\n `forcePromote` to the current rollout identifier: in `v1`, the value of\n `status.requestedRolloutHash`; in `v1alpha1`, the `requestRollout` value\n in the spec.\n \n To minimize downtime, promotion should occur when the new generation\n has caught up to the prior generation. To determine if the new\n generation has caught up, consult the `UpToDate` condition in the\n status of the Materialize Resource. If the condition's reason is\n `ReadyToPromote` the new generation is ready to promote.\n \n {{}}\n Do not leave new generations unpromoted indefinitely.\n \n The new generation keeps open read holds which prevent compaction. Once promoted or\n cancelled, those read holds are released. If left unpromoted for an extended time, this\n data can build up, and can cause extreme deletion load on the metadata backend database\n when finally promoted or cancelled.\n \n To guard against this, a rollout that remains in progress longer\n than `rolloutRequestTimeout` (default 24h) is automatically\n cancelled.\n {{}}\n- `ImmediatelyPromoteCausingDowntime`:
      {{}}\n THIS WILL CAUSE YOUR MATERIALIZE INSTANCE TO BE UNAVAILABLE FOR SOME TIME!!!\n \n This strategy should ONLY be used by customers with physical hardware who do not have\n enough hardware for the `WaitUntilReady` strategy. If you think you want this, please\n consult with Materialize engineering to discuss your situation.\n {{}}\n \n Tear down the old generation of pods and promote the new generation of pods immediately,\n without waiting for the new generation of pods to be ready.", + "default": "WaitUntilReady", + "required": false, + "deprecated": false + }, + { + "name": "serviceAccountAnnotations", + "type": "Map", + "description": "Annotations to apply to the service account.\n\nAnnotations on service accounts are commonly used by cloud providers for IAM.\nAWS uses \"eks.amazonaws.com/role-arn\".\nAzure uses \"azure.workload.identity/client-id\", but\nadditionally requires \"azure.workload.identity/use\": \"true\" on the pods.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "serviceAccountLabels", + "type": "Map", + "description": "Labels to apply to the service account.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "serviceAccountName", + "type": "String", + "description": "Name of the kubernetes service account to use.\nIf not set, we will create one with the same name as this Materialize object.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "systemParameterConfigmapName", + "type": "String", + "description": "The name of a ConfigMap containing system parameters in JSON format.\nThe ConfigMap must contain a `system-params.json` key whose value\nis a valid JSON object containing valid system parameters.\n\nRun `SHOW ALL` in SQL to see a subset of configurable system parameters.\n\nExample ConfigMap:\n```yaml\ndata:\n system-params.json: |\n {\n \"max_connections\": 1000\n }\n```", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "MaterializeCertSpec", + [ + { + "name": "dnsNames", + "type": "Array", + "description": "Additional DNS names the certificate will be valid for.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "duration", + "type": "String", + "description": "Duration the certificate will be requested for.\nValue must be in units accepted by Go\n[`time.ParseDuration`](https://golang.org/pkg/time/#ParseDuration).", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "issuerRef", + "type": "CertificateIssuerRef", + "description": "Reference to an `Issuer` or `ClusterIssuer` that will generate the certificate.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "privateKeyAlgorithm", + "type": "CertificatePrivateKeyAlgorithm", + "description": "Optional algorithm to use for the private key. If not specified, a recommended default will be chosen.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "privateKeySize", + "type": "Integer", + "description": "Optional size for the private key.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "renewBefore", + "type": "String", + "description": "Duration before expiration the certificate will be renewed.\nValue must be in units accepted by Go\n[`time.ParseDuration`](https://golang.org/pkg/time/#ParseDuration).", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "secretTemplate", + "type": "CertificateSecretTemplate", + "description": "Additional annotations and labels to include in the Certificate object.", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "CertificateSecretTemplate", + [ + { + "name": "annotations", + "type": "Map", + "description": "Annotations is a key value map to be copied to the target Kubernetes Secret.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "labels", + "type": "Map", + "description": "Labels is a key value map to be copied to the target Kubernetes Secret.", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "CertificateIssuerRef", + [ + { + "name": "name", + "type": "String", + "description": "Name of the resource being referred to.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "group", + "type": "String", + "description": "Group of the resource being referred to.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "kind", + "type": "String", + "description": "Kind of the resource being referred to.", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.ResourceRequirements", + [ + { + "name": "claims", + "type": "Array", + "description": "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container.\n\nThis is an alpha field and requires enabling the DynamicResourceAllocation feature gate.\n\nThis field is immutable. It can only be set for containers.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "limits", + "type": "Map", + "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "requests", + "type": "Map", + "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.ResourceClaim", + [ + { + "name": "name", + "type": "String", + "description": "Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "request", + "type": "String", + "description": "Request is the name chosen for a request in the referenced claim. If empty, everything from the claim is made available, otherwise only the result of this request.", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.EnvVar", + [ + { + "name": "name", + "type": "String", + "description": "Name of the environment variable. Must be a C_IDENTIFIER.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "value", + "type": "String", + "description": "Variable references $(VAR_NAME) are expanded using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "valueFrom", + "type": "io.k8s.api.core.v1.EnvVarSource", + "description": "Source for the environment variable's value. Cannot be used if value is not empty.", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.EnvVarSource", + [ + { + "name": "configMapKeyRef", + "type": "io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "fieldRef", + "type": "io.k8s.api.core.v1.ObjectFieldSelector", + "description": "Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "resourceFieldRef", + "type": "io.k8s.api.core.v1.ResourceFieldSelector", + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "secretKeyRef", + "type": "io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.SecretKeySelector", + [ + { + "name": "key", + "type": "String", + "description": "The key of the secret to select from. Must be a valid secret key.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "name", + "type": "String", + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "optional", + "type": "Bool", + "description": "Specify whether the Secret or its key must be defined", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.ResourceFieldSelector", + [ + { + "name": "resource", + "type": "String", + "description": "Required: resource to select", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "containerName", + "type": "String", + "description": "Container name: required for volumes, optional for env vars", + "default": null, + "required": false, + "deprecated": false + }, + { + "name": "divisor", + "type": "io.k8s.apimachinery.pkg.api.resource.Quantity", + "description": "Specifies the output format of the exposed resources, defaults to \"1\"", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.ObjectFieldSelector", + [ + { + "name": "fieldPath", + "type": "String", + "description": "Path of the field to select in the specified API version.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "apiVersion", + "type": "String", + "description": "Version of the schema the FieldPath is written in terms of, defaults to \"v1\".", + "default": null, + "required": false, + "deprecated": false + } + ] + ], + [ + "io.k8s.api.core.v1.ConfigMapKeySelector", + [ + { + "name": "key", + "type": "String", + "description": "The key to select.", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "name", + "type": "String", + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "default": null, + "required": true, + "deprecated": false + }, + { + "name": "optional", + "type": "Bool", + "description": "Specify whether the ConfigMap or its key must be defined", + "default": null, + "required": false, + "deprecated": false + } + ] + ] +] diff --git a/doc/user/data/self_managed/materialize_crd_descriptions.json b/doc/user/data/self_managed/materialize_crd_descriptions_v1alpha1.json similarity index 93% rename from doc/user/data/self_managed/materialize_crd_descriptions.json rename to doc/user/data/self_managed/materialize_crd_descriptions_v1alpha1.json index a87d9961edd19..4d080e4905910 100644 --- a/doc/user/data/self_managed/materialize_crd_descriptions.json +++ b/doc/user/data/self_managed/materialize_crd_descriptions_v1alpha1.json @@ -189,7 +189,7 @@ { "name": "rolloutStrategy", "type": "Enum", - "description": "Rollout strategy to use when upgrading this Materialize instance.\n\nValid values:\n- `WaitUntilReady` (default):
      Create a new generation of pods, leaving the old generation around until the\n new ones are ready to take over.\n This minimizes downtime, and is what almost everyone should use.\n- `ManuallyPromote`:
      Create a new generation of pods, leaving the old generation as the serving generation\n until the user manually promotes the new generation.\n \n When using `ManuallyPromote`, the new generation can be promoted at any\n time, even if it has dataflows that are not fully caught up, by setting\n `forcePromote` to the same value as `requestRollout` in the Materialize spec.\n \n To minimize downtime, promotion should occur when the new generation\n has caught up to the prior generation. To determine if the new\n generation has caught up, consult the `UpToDate` condition in the\n status of the Materialize Resource. If the condition's reason is\n `ReadyToPromote` the new generation is ready to promote.\n \n {{}}\n Do not leave new generations unpromoted indefinitely.\n \n The new generation keeps open read holds which prevent compaction. Once promoted or\n cancelled, those read holds are released. If left unpromoted for an extended time, this\n data can build up, and can cause extreme deletion load on the metadata backend database\n when finally promoted or cancelled.\n \n To guard against this, a rollout that remains in progress longer\n than `rolloutRequestTimeout` (default 24h) is automatically\n cancelled.\n {{}}\n- `ImmediatelyPromoteCausingDowntime`:
      {{}}\n THIS WILL CAUSE YOUR MATERIALIZE INSTANCE TO BE UNAVAILABLE FOR SOME TIME!!!\n \n This strategy should ONLY be used by customers with physical hardware who do not have\n enough hardware for the `WaitUntilReady` strategy. If you think you want this, please\n consult with Materialize engineering to discuss your situation.\n {{}}\n \n Tear down the old generation of pods and promote the new generation of pods immediately,\n without waiting for the new generation of pods to be ready.", + "description": "Rollout strategy to use when upgrading this Materialize instance.\n\nValid values:\n- `WaitUntilReady` (default):
      Create a new generation of pods, leaving the old generation around until the\n new ones are ready to take over.\n This minimizes downtime, and is what almost everyone should use.\n- `ManuallyPromote`:
      Create a new generation of pods, leaving the old generation as the serving generation\n until the user manually promotes the new generation.\n \n When using `ManuallyPromote`, the new generation can be promoted at any\n time, even if it has dataflows that are not fully caught up, by setting\n `forcePromote` to the current rollout identifier: in `v1`, the value of\n `status.requestedRolloutHash`; in `v1alpha1`, the `requestRollout` value\n in the spec.\n \n To minimize downtime, promotion should occur when the new generation\n has caught up to the prior generation. To determine if the new\n generation has caught up, consult the `UpToDate` condition in the\n status of the Materialize Resource. If the condition's reason is\n `ReadyToPromote` the new generation is ready to promote.\n \n {{}}\n Do not leave new generations unpromoted indefinitely.\n \n The new generation keeps open read holds which prevent compaction. Once promoted or\n cancelled, those read holds are released. If left unpromoted for an extended time, this\n data can build up, and can cause extreme deletion load on the metadata backend database\n when finally promoted or cancelled.\n \n To guard against this, a rollout that remains in progress longer\n than `rolloutRequestTimeout` (default 24h) is automatically\n cancelled.\n {{}}\n- `ImmediatelyPromoteCausingDowntime`:
      {{}}\n THIS WILL CAUSE YOUR MATERIALIZE INSTANCE TO BE UNAVAILABLE FOR SOME TIME!!!\n \n This strategy should ONLY be used by customers with physical hardware who do not have\n enough hardware for the `WaitUntilReady` strategy. If you think you want this, please\n consult with Materialize engineering to discuss your situation.\n {{}}\n \n Tear down the old generation of pods and promote the new generation of pods immediately,\n without waiting for the new generation of pods to be ready.", "default": "WaitUntilReady", "required": false, "deprecated": false diff --git a/doc/user/data/self_managed/upgrades.yml b/doc/user/data/self_managed/upgrades.yml index d53015f6ee8d2..39eb0b71e11b6 100644 --- a/doc/user/data/self_managed/upgrades.yml +++ b/doc/user/data/self_managed/upgrades.yml @@ -2,8 +2,13 @@ content: | When upgrading: - - **Always** check the [version specific upgrade - notes](/self-managed-deployments/upgrading/#version-specific-upgrade-notes). + - **Always** check the [version-specific upgrade + notes](/self-managed-deployments/upgrading/version-notes/) for your target + version: + - [Upgrading to `v26.30` and later versions](/self-managed-deployments/upgrading/version-notes/#upgrading-to-v2630-and-later-versions) + - [Upgrading to `v26.1` and later versions](/self-managed-deployments/upgrading/version-notes/#upgrading-to-v261-and-later-versions) + - [Upgrading to `v26.0`](/self-managed-deployments/upgrading/version-notes/#upgrading-to-v260) + - [Upgrading between minor versions less than `v26`](/self-managed-deployments/upgrading/version-notes/#upgrading-between-minor-versions-less-than-v26) - {{< include-from-yaml data="self_managed/upgrades" name="upgrade-order-rule" >}} @@ -131,12 +136,40 @@ name="upgrade-major-version-restriction" %}} {{< /note >}} + Select the tab for the CRD API version your deployment currently uses. + This step does not cover migrating from `v1alpha1` to `v1`; to migrate, + see [Adopting the v1 + CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). (The `v1` + CRD API version is available starting in Materialize v26.30 / Terraform + module v3.0.15.) + +
      +
        +
        +
        + + {{< self-managed/crd-version-note "v1alpha1" >}} + ```shell helm upgrade -n materialize simple-demo materialize/materialize-operator \ -f my-values.yaml \ --version {{< self-managed/versions/get-latest-version >}} ``` +
        +
        + + ```shell + helm upgrade -n materialize simple-demo materialize/materialize-operator \ + -f my-values.yaml \ + --version {{< self-managed/versions/get-latest-version >}} \ + --set operator.args.installV1CRD=true + ``` + +
        +
        +
        + 1. Verify that the Operator is running: ```bash @@ -156,18 +189,32 @@ content: | **After** you have upgraded your Materialize Operator, upgrade your - Materialize instance(s) to the **APP Version** of the Operator. When - upgrading Materialize Instances versions, changes are not automatically - rolled out by the Operator in order to minimize unexpected downtime and - avoid connection drops at critical periods. Instead, the upgrade process - involves two steps: + Materialize instance(s) to the **APP Version** of the Operator. + + Select the tab for the CRD API version your deployment currently uses. This + step does not cover migrating from `v1alpha1` to `v1`; to migrate, see + [Adopting the v1 + CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). (The `v1` + CRD API version is available starting in Materialize v26.30 / Terraform + module v3.0.15.) + +
        +
          +
          +
          + + {{< self-managed/crd-version-note "v1alpha1" >}} + + With `v1alpha1`, changes are not automatically rolled out by the Operator in + order to minimize unexpected downtime and avoid connection drops at critical + periods. Instead, the upgrade process involves two steps: - First, staging the version change to the Materialize custom resource. - Second, rolling out the changes via a `requestRollout` flag. 1. Find the name of the Materialize instance to upgrade. The sample example - deployment using the unified Terraform module deploys the Materialie - instance in the`materialize-environment` namespace. + deployment using the unified Terraform module deploys the Materialize + instance in the `materialize-environment` namespace. ```shell kubectl get materialize -n materialize-environment @@ -183,7 +230,7 @@ 1. Stage, but not rollout, the Materialize instance version upgrade. ```shell - kubectl patch materialize main\ + kubectl patch materialize main \ -n materialize-environment \ --type='merge' \ -p "{\"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\"}}" @@ -203,3 +250,43 @@ ```bash kubectl -n materialize-environment describe pod -l app=environmentd ``` + +
          +
          + + With `v1`, updating the spec automatically triggers a rollout, + so no `requestRollout` is needed. + + 1. Find the name of the Materialize instance to upgrade. The sample example + deployment using the unified Terraform module deploys the Materialize + instance in the `materialize-environment` namespace. + + ```shell + kubectl get materialize -n materialize-environment + ``` + + In the example deployment, the name of the instance is `main`. + + ```none + NAME + main + ``` + + 1. Apply the Materialize instance version upgrade. + + ```shell + kubectl patch materialize main \ + -n materialize-environment \ + --type='merge' \ + -p "{\"spec\": {\"environmentdImageRef\": \"docker.io/materialize/environmentd:{{< self-managed/versions/get-latest-version >}}\"}}" + ``` + + 1. Verify the upgrade by checking the `environmentd` events: + + ```bash + kubectl -n materialize-environment describe pod -l app=environmentd + ``` + +
          +
          +
          diff --git a/doc/user/layouts/partials/self-managed/crd-version-note.html b/doc/user/layouts/partials/self-managed/crd-version-note.html new file mode 100644 index 0000000000000..37827002f2402 --- /dev/null +++ b/doc/user/layouts/partials/self-managed/crd-version-note.html @@ -0,0 +1,13 @@ +{{- /* CRD version blurb for tabbed v1alpha1/v1 examples. + Arg: the version string, "v1alpha1" or "v1". */ -}} +{{- $v := . -}} +{{- /* Prepend the docs base path (e.g. /docs) the same way render-link.html + does for markdown links, so the raw resolves under the deployed + subdirectory. */ -}} +{{- $base := strings.TrimRight "/" (urls.Parse site.BaseURL).Path -}} +{{- $adopt := printf "%s/self-managed-deployments/upgrading/adopting-the-v1-crd/" $base -}} +{{- if eq $v "v1alpha1" -}} +

          v1alpha1 is the default CRD version. With v1alpha1, rollouts require manually rotating a UUID. For a simplified rollout behavior, see Adopting the v1 CRD.

          +{{- else -}} +

          The v1 CRD is available starting in v26.30 and is opt-in. See Adopting the v1 CRD to enable it.

          +{{- end -}} diff --git a/doc/user/layouts/shortcodes/self-managed/crd-version-note.html b/doc/user/layouts/shortcodes/self-managed/crd-version-note.html new file mode 100644 index 0000000000000..52b914283f852 --- /dev/null +++ b/doc/user/layouts/shortcodes/self-managed/crd-version-note.html @@ -0,0 +1 @@ +{{- partial "self-managed/crd-version-note.html" (.Get 0) -}} diff --git a/doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions.html b/doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions-v1.html similarity index 89% rename from doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions.html rename to doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions-v1.html index 5c09e0cb47f88..66b194694e322 100644 --- a/doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions.html +++ b/doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions-v1.html @@ -1,10 +1,10 @@ {{ $types := dict }} -{{ range $.Site.Data.self_managed.materialize_crd_descriptions }} +{{ range $.Site.Data.self_managed.materialize_crd_descriptions_v1 }} {{ $types = merge $types (dict (index . 0) true) }} {{ end }} -{{ range $.Site.Data.self_managed.materialize_crd_descriptions }} +{{ range $.Site.Data.self_managed.materialize_crd_descriptions_v1 }} #### {{ index . 0 }} diff --git a/doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions-v1alpha1.html b/doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions-v1alpha1.html new file mode 100644 index 0000000000000..de77f8a4e1a95 --- /dev/null +++ b/doc/user/layouts/shortcodes/self-managed/materialize-crd-descriptions-v1alpha1.html @@ -0,0 +1,57 @@ + +{{ $types := dict }} +{{ range $.Site.Data.self_managed.materialize_crd_descriptions_v1alpha1 }} +{{ $types = merge $types (dict (index . 0) true) }} +{{ end }} + +{{ range $.Site.Data.self_managed.materialize_crd_descriptions_v1alpha1 }} +#### {{ index . 0 }} +
          + + + + + + + + +{{- range (index . 1) }} + {{- if not .deprecated }} + + + + + + {{- end }} +{{- end }} + +
          Field NameRequiredDescription
          {{ .name }}{{ if .required }}✅{{ end }} + + {{- $matches := findRESubmatch `(Array<|Map]+)(>*)` .type 1 -}} + {{- $captures := index $matches 0 -}} + + {{- $typePrefix := index $captures 1 -}} + {{- $baseType := index $captures 2 -}} + {{- $typeSuffix := index $captures 3 -}} + + {{- $typePrefix -}} + {{- if (index $types $baseType) -}} + + {{- end -}} + {{ $baseType }} + {{- if (index $types $baseType) -}} + + {{- end -}} + {{- $typeSuffix -}} + + + +

          {{- .description | markdownify -}}

          + + + {{- if .default -}} +

          Default: {{ .default }}

          + {{- end -}} + +
          +{{ end }} diff --git a/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html b/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html index 249c9b2ec3ca0..353098515966d 100644 --- a/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html +++ b/doc/user/layouts/shortcodes/self-managed/versions/step-install-helm-version-local-kind-install.html @@ -3,10 +3,24 @@ 1. Install the Materialize Operator. The operator will be installed in the `materialize` namespace. + Starting in v26.30, you can choose between CRD versions + [`v1`](/self-managed-deployments/upgrading/version-notes/#upgrading-to-v2630-and-later-versions) + and `v1alpha1`. The following enables the `v1` CRD version by setting + `operator.args.installV1CRD=true`. The `v1` CRD provides a [simplified + rollout + behavior](/self-managed-deployments/upgrading/version-notes/#upgrading-to-v2630-and-later-versions), + but requires `cert-manager`, installed in an [earlier + step](#install-cert-manager) in this guide. + ```shell helm install my-materialize-operator materialize/materialize-operator \ --namespace=materialize --create-namespace \ --version {{ $operator_version }} \ --set observability.podMetrics.enabled=true \ + --set operator.args.installV1CRD=true \ -f sample-values.yaml ``` + + To remain on the default `v1alpha1` CRD version, omit `--set + operator.args.installV1CRD=true`. With `v1alpha1`, rollouts require manually + rotating a UUID. diff --git a/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html b/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html index 94ee296c4aac3..085dd74db25ab 100644 --- a/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html +++ b/doc/user/layouts/shortcodes/self-managed/versions/upgrade/upgrade-steps-local-kind.html @@ -30,6 +30,19 @@ configuration file. Include any additional configurations that you specify for your deployment. + Select the tab for the CRD API version your deployment currently uses. This + step does not cover migrating from `v1alpha1` to `v1`; to migrate, see + [Adopting the v1 + CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). (The `v1` CRD + API version is available starting in Materialize v26.30.) + +
          +
            +
            +
            + + {{ partial "self-managed/crd-version-note.html" "v1alpha1" }} + ```shell helm upgrade my-materialize-operator materialize/materialize-operator \ --namespace=materialize \ @@ -38,6 +51,27 @@ --set observability.podMetrics.enabled=true ``` +
            +
            + + Enabling the `v1` CRD requires cert-manager (or your own certificate) and + API-server ingress to the operator on port `8001`. If you have not already + installed cert-manager, install it first (see the [Install on + kind](/self-managed-deployments/installation/install-on-local-kind/) guide). + + ```shell + helm upgrade my-materialize-operator materialize/materialize-operator \ + --namespace=materialize \ + --version {{ $operator_version }} \ + -f upgrade-values.yaml \ + --set observability.podMetrics.enabled=true \ + --set operator.args.installV1CRD=true + ``` + +
            +
            +
            + 1. Verify that the operator is running: ```bash @@ -63,6 +97,19 @@ 1. Create a new `upgrade-materialize.yaml` file, updating the following fields: + Select the tab for the CRD API version your deployment currently uses. This + step does not cover migrating from `v1alpha1` to `v1`; to migrate, see + [Adopting the v1 + CRD](/self-managed-deployments/upgrading/adopting-the-v1-crd/). (The `v1` CRD + API version is available starting in Materialize v26.30.) + +
            +
              +
              +
              + + If using `v1alpha1`, update the following fields: + | Field | Description | |-------|-------------| | `environmentdImageRef` | Update the version to the new version. This should be the same as the operator version: `{{ $operator_version }}`. | @@ -83,6 +130,34 @@ backendSecretName: materialize-backend ``` +
              +
              + + If using `v1`, update the following fields: + + | Field | Description | + |-------|-------------| + | `environmentdImageRef` | Update the version to the new version. This should be the same as the operator version: `{{ $operator_version }}`. Updating this field automatically triggers a rollout. | + | `forceRollout` | **Optional.** Set to a new UUID (can be generated with `uuidgen`) only to force a rollout when no other changes exist. | + + + ```yaml + apiVersion: materialize.cloud/v1 + kind: Materialize + metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment + spec: + environmentdImageRef: materialize/environmentd:{{ $environmentd_version }} # Update version + # forceRollout: 33333333-3333-3333-3333-333333333333 # For forced rollouts + rolloutStrategy: WaitUntilReady # The mechanism to use when rolling out the new version. + backendSecretName: materialize-backend + ``` + +
              +
              +
              + 1. Apply the upgrade-materialize.yaml file to your Materialize instance: ```shell diff --git a/doc/user/shared-content/self-managed/general-rules-for-upgrades.md b/doc/user/shared-content/self-managed/general-rules-for-upgrades.md index 86f661ddcde23..c10d4e0b4fb0e 100644 --- a/doc/user/shared-content/self-managed/general-rules-for-upgrades.md +++ b/doc/user/shared-content/self-managed/general-rules-for-upgrades.md @@ -1,7 +1,7 @@ Whe upgrading: -- **Always** check the [version specific upgrade - notes](/self-managed-deployments/upgrading/#version-specific-upgrade-notes). +- **Always** check the [version-specific upgrade + notes](/self-managed-deployments/upgrading/version-notes/). - **Always** upgrade the operator **first** and ensure version compatibility between the operator and the Materialize instance you are upgrading to. diff --git a/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md b/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md new file mode 100644 index 0000000000000..c13fde5f471bc --- /dev/null +++ b/doc/user/shared-content/self-managed/upgrade-notes/v26.30.md @@ -0,0 +1,45 @@ +v26.30.0 introduces a new version of the Materialize CRD, `v1`. The new CRD simplifies rollouts by automatically detecting changes. This means you will no longer need to manually rotate a UUID to trigger a rollout. + +Before, on `v1alpha1`: +```yaml +apiVersion: materialize.cloud/v1alpha1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:v26.16.0 + requestRollout: 22222222-2222-2222-2222-222222222222 # ← MUST set a new UUID every upgrade +# forceRollout: 33333333-3333-3333-3333-333333333333 # ← for forced rollouts + rolloutStrategy: WaitUntilReady + backendSecretName: materialize-backend +``` + +After, on `v1`: +```yaml +apiVersion: materialize.cloud/v1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:v26.16.0 # ← just change this +# forceRollout: 33333333-3333-3333-3333-333333333333 # ← only for forced rollouts + rolloutStrategy: WaitUntilReady + backendSecretName: materialize-backend +``` + +With the new change, the `requestRollout` field will be removed, along with all previously deprecated fields. + +Upgrading to v26.30 does **not** require adopting the `v1` CRD; adopting `v1` is +**opt-in**. You can upgrade as usual while continuing to use `v1alpha1`; your existing instances will behave exactly as before. + +Adopting `v1` requires infrastructure changes (cert-manager or your own +certificate, and API-server ingress on port `8001`) and setting the Helm value +`operator.args.installV1CRD=true`; see [Adopting the v1 CRD +Prerequisites](/self-managed-deployments/upgrading/adopting-the-v1-crd/#prerequisites). + +{{< note >}} +Upgrading to v26.30 while remaining on `v1alpha1` requires **no** infrastructure +changes. +{{< /note >}} diff --git a/src/cloud-resources/src/bin/crd_writer.rs b/src/cloud-resources/src/bin/crd_writer.rs index 7a51c8386ba4e..b2355d5656751 100644 --- a/src/cloud-resources/src/bin/crd_writer.rs +++ b/src/cloud-resources/src/bin/crd_writer.rs @@ -8,9 +8,11 @@ // by the Apache License, Version 2.0. use std::cmp::Ordering; +use std::env; use indexmap::{IndexMap, IndexSet}; -use mz_cloud_resources::crd::materialize::v1alpha1::MaterializeSpec; +use mz_cloud_resources::crd::materialize::v1; +use mz_cloud_resources::crd::materialize::v1alpha1; use schemars::schema_for; use serde::Serialize; @@ -328,7 +330,14 @@ fn format_enum_variants_from_json( } fn main() { - let root_schema = schema_for!(MaterializeSpec); + let args: Vec = env::args().collect(); + let version = args.get(1).expect("usage: crd-writer "); + + let root_schema = match version.as_str() { + "v1alpha1" => schema_for!(v1alpha1::MaterializeSpec), + "v1" => schema_for!(v1::MaterializeSpec), + other => panic!("unknown version: {other}, expected v1alpha1 or v1"), + }; // Convert all to JSON for easier merging let schema_json = root_schema.to_value(); diff --git a/src/cloud-resources/src/crd/materialize.rs b/src/cloud-resources/src/crd/materialize.rs index afe22d675accd..2729cef44e82a 100644 --- a/src/cloud-resources/src/crd/materialize.rs +++ b/src/cloud-resources/src/crd/materialize.rs @@ -45,7 +45,9 @@ pub enum MaterializeRolloutStrategy { /// /// When using `ManuallyPromote`, the new generation can be promoted at any /// time, even if it has dataflows that are not fully caught up, by setting - /// `forcePromote` to the same value as `requestRollout` in the Materialize spec. + /// `forcePromote` to the current rollout identifier: in `v1`, the value of + /// `status.requestedRolloutHash`; in `v1alpha1`, the `requestRollout` value + /// in the spec. /// /// To minimize downtime, promotion should occur when the new generation /// has caught up to the prior generation. To determine if the new @@ -970,7 +972,7 @@ pub mod v1 { /// operator cancels the rollout once this timeout is exceeded: the new /// generation is torn down and the previously-active generation /// continues serving. A new rollout can then be triggered by setting - /// `requestRollout` to a new value. + /// `forceRollout` to a new value. /// /// This does not apply to the `ImmediatelyPromoteCausingDowntime` /// rollout strategy or to force-promoted rollouts, since by the time