From bd9fa60891f45f858d4b5caa4910f6816cf2a19d Mon Sep 17 00:00:00 2001 From: ali Date: Tue, 16 Jun 2026 11:42:56 -0700 Subject: [PATCH 1/2] refactor(k8s): organize Helm templates into common/aws/onprem Group the chart's templates by where they apply so the layout makes the deployment surface obvious at a glance: templates/common/ resources every deployment needs (services, gateway, postgres/lakefs/lakekeeper, computing-unit pool, RBAC) templates/onprem/ self-hosted-only resources (in-cluster MinIO) templates/aws/ AWS/EKS-only resources (added in later PRs; placeholder) Helm renders templates/** recursively, so this is purely organizational: `helm template` output is byte-identical to before the move (verified, modulo the chart's pre-existing per-render random LakeFS keys). A templates/README.md documents the convention and a .helmignore keeps the doc and .gitkeep placeholder from being loaded as manifests. Also bring values-development.yaml back in line with values.yaml (the ground truth), which had drifted: - image source: docker.io/apache + 1.3.0-incubating-SNAPSHOT (was the stale ghcr.io/apache + latest), so both value files pull from the same place; - add the AUTH_JWT_SECRET entry to texeraEnvVars. values.yaml has it but the dev profile omitted it, so the computing-unit manager started without the secret and k8s computing-unit creation crashed with NoSuchElementException (None.get) in ComputingUnitManagingResource. Adding it (same dev-only default as values.yaml) makes CU creation work under the dev profile. No behavior change to the default (on-prem) install. Co-Authored-By: Claude Opus 4.8 (1M context) --- bin/k8s/.helmignore | 17 +++++ bin/k8s/templates/README.md | 68 +++++++++++++++++++ bin/k8s/templates/aws/.gitkeep | 2 + .../access-control-service-deployment.yaml | 0 .../access-control-service-service.yaml | 0 .../agent-service-backend-traffic-policy.yaml | 0 .../agent-service-deployment.yaml | 0 .../agent-service}/agent-service-secret.yaml | 0 .../agent-service}/agent-service-service.yaml | 0 .../config-service-deployment.yaml | 0 .../config-service-service.yaml | 0 .../example-data-loader-job.yaml | 0 .../external-names}/external-names.yaml | 0 .../file-service-deployment.yaml | 0 .../file-service}/file-service-service.yaml | 0 .../{ => common/gateway}/gateway-backend.yaml | 0 .../{ => common/gateway}/gateway-routes.yaml | 0 .../gateway}/gateway-security-policy.yaml | 0 .../{ => common/gateway}/gateway.yaml | 0 .../{ => common/lakefs}/lakefs-secret.yaml | 0 .../{ => common/lakefs}/lakefs-setup-job.yaml | 0 .../lakekeeper}/lakekeeper-init-job.yaml | 0 .../{ => common/litellm}/litellm-config.yaml | 0 .../litellm}/litellm-deployment.yaml | 0 .../{ => common/litellm}/litellm-service.yaml | 0 .../postgresql-init-script-config.yaml | 0 .../postgresql}/postgresql-persistence.yaml | 0 .../templates/{ => common/pylsp}/pylsp.yaml | 0 .../shared-editing-server.yaml | 0 .../webserver}/webserver-deployment.yaml | 0 .../webserver}/webserver-service.yaml | 0 ...workflow-compiling-service-deployment.yaml | 0 .../workflow-compiling-service-service.yaml | 0 ...low-computing-unit-manager-deployment.yaml | 0 ...omputing-unit-manager-service-account.yaml | 0 ...rkflow-computing-unit-manager-service.yaml | 0 ...mputing-unit-master-prepull-daemonset.yaml | 0 ...orkflow-computing-unit-resource-quota.yaml | 0 .../workflow-computing-units-namespace.yaml | 0 .../workflow-computing-units-service.yaml | 0 .../{ => onprem}/minio-persistence.yaml | 0 bin/k8s/values-development.yaml | 7 +- 42 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 bin/k8s/.helmignore create mode 100644 bin/k8s/templates/README.md create mode 100644 bin/k8s/templates/aws/.gitkeep rename bin/k8s/templates/{ => common/access-control-service}/access-control-service-deployment.yaml (100%) rename bin/k8s/templates/{ => common/access-control-service}/access-control-service-service.yaml (100%) rename bin/k8s/templates/{ => common/agent-service}/agent-service-backend-traffic-policy.yaml (100%) rename bin/k8s/templates/{ => common/agent-service}/agent-service-deployment.yaml (100%) rename bin/k8s/templates/{ => common/agent-service}/agent-service-secret.yaml (100%) rename bin/k8s/templates/{ => common/agent-service}/agent-service-service.yaml (100%) rename bin/k8s/templates/{ => common/config-service}/config-service-deployment.yaml (100%) rename bin/k8s/templates/{ => common/config-service}/config-service-service.yaml (100%) rename bin/k8s/templates/{ => common/example-data-loader}/example-data-loader-job.yaml (100%) rename bin/k8s/templates/{ => common/external-names}/external-names.yaml (100%) rename bin/k8s/templates/{ => common/file-service}/file-service-deployment.yaml (100%) rename bin/k8s/templates/{ => common/file-service}/file-service-service.yaml (100%) rename bin/k8s/templates/{ => common/gateway}/gateway-backend.yaml (100%) rename bin/k8s/templates/{ => common/gateway}/gateway-routes.yaml (100%) rename bin/k8s/templates/{ => common/gateway}/gateway-security-policy.yaml (100%) rename bin/k8s/templates/{ => common/gateway}/gateway.yaml (100%) rename bin/k8s/templates/{ => common/lakefs}/lakefs-secret.yaml (100%) rename bin/k8s/templates/{ => common/lakefs}/lakefs-setup-job.yaml (100%) rename bin/k8s/templates/{ => common/lakekeeper}/lakekeeper-init-job.yaml (100%) rename bin/k8s/templates/{ => common/litellm}/litellm-config.yaml (100%) rename bin/k8s/templates/{ => common/litellm}/litellm-deployment.yaml (100%) rename bin/k8s/templates/{ => common/litellm}/litellm-service.yaml (100%) rename bin/k8s/templates/{ => common/postgresql}/postgresql-init-script-config.yaml (100%) rename bin/k8s/templates/{ => common/postgresql}/postgresql-persistence.yaml (100%) rename bin/k8s/templates/{ => common/pylsp}/pylsp.yaml (100%) rename bin/k8s/templates/{ => common/shared-editing-server}/shared-editing-server.yaml (100%) rename bin/k8s/templates/{ => common/webserver}/webserver-deployment.yaml (100%) rename bin/k8s/templates/{ => common/webserver}/webserver-service.yaml (100%) rename bin/k8s/templates/{ => common/workflow-compiling-service}/workflow-compiling-service-deployment.yaml (100%) rename bin/k8s/templates/{ => common/workflow-compiling-service}/workflow-compiling-service-service.yaml (100%) rename bin/k8s/templates/{ => common/workflow-computing-unit-manager}/workflow-computing-unit-manager-deployment.yaml (100%) rename bin/k8s/templates/{ => common/workflow-computing-unit-manager}/workflow-computing-unit-manager-service-account.yaml (100%) rename bin/k8s/templates/{ => common/workflow-computing-unit-manager}/workflow-computing-unit-manager-service.yaml (100%) rename bin/k8s/templates/{ => common/workflow-computing-unit-pool}/workflow-computing-unit-master-prepull-daemonset.yaml (100%) rename bin/k8s/templates/{ => common/workflow-computing-unit-pool}/workflow-computing-unit-resource-quota.yaml (100%) rename bin/k8s/templates/{ => common/workflow-computing-unit-pool}/workflow-computing-units-namespace.yaml (100%) rename bin/k8s/templates/{ => common/workflow-computing-unit-pool}/workflow-computing-units-service.yaml (100%) rename bin/k8s/templates/{ => onprem}/minio-persistence.yaml (100%) diff --git a/bin/k8s/.helmignore b/bin/k8s/.helmignore new file mode 100644 index 00000000000..9577bcfcd70 --- /dev/null +++ b/bin/k8s/.helmignore @@ -0,0 +1,17 @@ +# Patterns to ignore when building Helm packages. +# Operating system / editor files +.DS_Store +*.swp +*.bak +*.tmp +*.orig +*~ +# Version control directories +.git/ +.gitignore +# Documentation kept alongside the chart for humans, not rendered as manifests. +# Markdown under templates/ (e.g. the common/aws/onprem layout guide) must be +# ignored so Helm does not try to load it as a Kubernetes manifest. +*.md +# git placeholders must not be loaded by Helm as manifests +.gitkeep diff --git a/bin/k8s/templates/README.md b/bin/k8s/templates/README.md new file mode 100644 index 00000000000..bdf12287de4 --- /dev/null +++ b/bin/k8s/templates/README.md @@ -0,0 +1,68 @@ + + +# Helm template layout + +Templates are grouped by **where they apply**, so a reader can tell at a glance +which resources are deployment-agnostic and which are tied to a particular +hosting environment. Helm renders every file under `templates/` recursively, so +these subdirectories are purely organizational — they do not change rendering. + +| Folder | Contains | Renders when | +|--------|----------|--------------| +| `common/` | Resources every deployment needs: the Texera micro-service Deployments/Services, the Envoy Gateway + routes, Postgres/LakeFS/Lakekeeper wiring, the computing-unit pool, RBAC and namespaces. | Always. | +| `onprem/` | Resources only used by a self-hosted / local deployment, e.g. the in-cluster MinIO persistence. | Gated on the relevant on-prem value (e.g. `minio.enabled`). | +| `aws/` | Resources only used on AWS/EKS, e.g. the external-S3 credentials Secret, the AWS NLB/EIP `EnvoyProxy`, and the autoscaler warm-pool placeholder. | Gated so they render to nothing off AWS (empty by default). | + +Within `common/`, templates are further grouped into one subfolder per +component, named after the service it belongs to, so every manifest for a given +piece sits together: + +``` +common/ + access-control-service/ # access-control-service Deployment + Service + agent-service/ # agent-service Deployment + Service + Secret + traffic policy + config-service/ + file-service/ + gateway/ # Envoy Gateway + routes + backends + security policy + lakefs/ + lakekeeper/ + litellm/ + postgresql/ # in-cluster Postgres PV/PVC + init scripts + webserver/ # dashboard / webserver Deployment + Service + workflow-compiling-service/ + workflow-computing-unit-manager/ # the CU manager service + RBAC + workflow-computing-unit-pool/ # CU pool namespace, quota, prepull, service + example-data-loader/ + external-names/ # ExternalName service aliases + pylsp/ # python language server + shared-editing-server/ # y-websocket collaborative editing +``` + +This nesting is also purely organizational — Helm still renders every file +recursively. + +Guidelines for adding a template: +- Default to `common/`, in the subfolder for the component it belongs to (create + a new one if it is a new component). Most resources are shared; only move a + file out to `aws/`/`onprem/` when it is genuinely specific to one hosting + environment. +- Anything under `aws/` or `onprem/` **must** be guarded by an `{{- if ... }}` + on an opt-in value so that the default (on-prem) install is unaffected and an + AWS install does not pick up on-prem-only resources. diff --git a/bin/k8s/templates/aws/.gitkeep b/bin/k8s/templates/aws/.gitkeep new file mode 100644 index 00000000000..87fae5f6485 --- /dev/null +++ b/bin/k8s/templates/aws/.gitkeep @@ -0,0 +1,2 @@ +# Placeholder so the (initially empty) aws/ template folder is tracked in git. +# AWS-only, value-gated templates are added by later PRs in the AWS/EKS series. diff --git a/bin/k8s/templates/access-control-service-deployment.yaml b/bin/k8s/templates/common/access-control-service/access-control-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/access-control-service-deployment.yaml rename to bin/k8s/templates/common/access-control-service/access-control-service-deployment.yaml diff --git a/bin/k8s/templates/access-control-service-service.yaml b/bin/k8s/templates/common/access-control-service/access-control-service-service.yaml similarity index 100% rename from bin/k8s/templates/access-control-service-service.yaml rename to bin/k8s/templates/common/access-control-service/access-control-service-service.yaml diff --git a/bin/k8s/templates/agent-service-backend-traffic-policy.yaml b/bin/k8s/templates/common/agent-service/agent-service-backend-traffic-policy.yaml similarity index 100% rename from bin/k8s/templates/agent-service-backend-traffic-policy.yaml rename to bin/k8s/templates/common/agent-service/agent-service-backend-traffic-policy.yaml diff --git a/bin/k8s/templates/agent-service-deployment.yaml b/bin/k8s/templates/common/agent-service/agent-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/agent-service-deployment.yaml rename to bin/k8s/templates/common/agent-service/agent-service-deployment.yaml diff --git a/bin/k8s/templates/agent-service-secret.yaml b/bin/k8s/templates/common/agent-service/agent-service-secret.yaml similarity index 100% rename from bin/k8s/templates/agent-service-secret.yaml rename to bin/k8s/templates/common/agent-service/agent-service-secret.yaml diff --git a/bin/k8s/templates/agent-service-service.yaml b/bin/k8s/templates/common/agent-service/agent-service-service.yaml similarity index 100% rename from bin/k8s/templates/agent-service-service.yaml rename to bin/k8s/templates/common/agent-service/agent-service-service.yaml diff --git a/bin/k8s/templates/config-service-deployment.yaml b/bin/k8s/templates/common/config-service/config-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/config-service-deployment.yaml rename to bin/k8s/templates/common/config-service/config-service-deployment.yaml diff --git a/bin/k8s/templates/config-service-service.yaml b/bin/k8s/templates/common/config-service/config-service-service.yaml similarity index 100% rename from bin/k8s/templates/config-service-service.yaml rename to bin/k8s/templates/common/config-service/config-service-service.yaml diff --git a/bin/k8s/templates/example-data-loader-job.yaml b/bin/k8s/templates/common/example-data-loader/example-data-loader-job.yaml similarity index 100% rename from bin/k8s/templates/example-data-loader-job.yaml rename to bin/k8s/templates/common/example-data-loader/example-data-loader-job.yaml diff --git a/bin/k8s/templates/external-names.yaml b/bin/k8s/templates/common/external-names/external-names.yaml similarity index 100% rename from bin/k8s/templates/external-names.yaml rename to bin/k8s/templates/common/external-names/external-names.yaml diff --git a/bin/k8s/templates/file-service-deployment.yaml b/bin/k8s/templates/common/file-service/file-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/file-service-deployment.yaml rename to bin/k8s/templates/common/file-service/file-service-deployment.yaml diff --git a/bin/k8s/templates/file-service-service.yaml b/bin/k8s/templates/common/file-service/file-service-service.yaml similarity index 100% rename from bin/k8s/templates/file-service-service.yaml rename to bin/k8s/templates/common/file-service/file-service-service.yaml diff --git a/bin/k8s/templates/gateway-backend.yaml b/bin/k8s/templates/common/gateway/gateway-backend.yaml similarity index 100% rename from bin/k8s/templates/gateway-backend.yaml rename to bin/k8s/templates/common/gateway/gateway-backend.yaml diff --git a/bin/k8s/templates/gateway-routes.yaml b/bin/k8s/templates/common/gateway/gateway-routes.yaml similarity index 100% rename from bin/k8s/templates/gateway-routes.yaml rename to bin/k8s/templates/common/gateway/gateway-routes.yaml diff --git a/bin/k8s/templates/gateway-security-policy.yaml b/bin/k8s/templates/common/gateway/gateway-security-policy.yaml similarity index 100% rename from bin/k8s/templates/gateway-security-policy.yaml rename to bin/k8s/templates/common/gateway/gateway-security-policy.yaml diff --git a/bin/k8s/templates/gateway.yaml b/bin/k8s/templates/common/gateway/gateway.yaml similarity index 100% rename from bin/k8s/templates/gateway.yaml rename to bin/k8s/templates/common/gateway/gateway.yaml diff --git a/bin/k8s/templates/lakefs-secret.yaml b/bin/k8s/templates/common/lakefs/lakefs-secret.yaml similarity index 100% rename from bin/k8s/templates/lakefs-secret.yaml rename to bin/k8s/templates/common/lakefs/lakefs-secret.yaml diff --git a/bin/k8s/templates/lakefs-setup-job.yaml b/bin/k8s/templates/common/lakefs/lakefs-setup-job.yaml similarity index 100% rename from bin/k8s/templates/lakefs-setup-job.yaml rename to bin/k8s/templates/common/lakefs/lakefs-setup-job.yaml diff --git a/bin/k8s/templates/lakekeeper-init-job.yaml b/bin/k8s/templates/common/lakekeeper/lakekeeper-init-job.yaml similarity index 100% rename from bin/k8s/templates/lakekeeper-init-job.yaml rename to bin/k8s/templates/common/lakekeeper/lakekeeper-init-job.yaml diff --git a/bin/k8s/templates/litellm-config.yaml b/bin/k8s/templates/common/litellm/litellm-config.yaml similarity index 100% rename from bin/k8s/templates/litellm-config.yaml rename to bin/k8s/templates/common/litellm/litellm-config.yaml diff --git a/bin/k8s/templates/litellm-deployment.yaml b/bin/k8s/templates/common/litellm/litellm-deployment.yaml similarity index 100% rename from bin/k8s/templates/litellm-deployment.yaml rename to bin/k8s/templates/common/litellm/litellm-deployment.yaml diff --git a/bin/k8s/templates/litellm-service.yaml b/bin/k8s/templates/common/litellm/litellm-service.yaml similarity index 100% rename from bin/k8s/templates/litellm-service.yaml rename to bin/k8s/templates/common/litellm/litellm-service.yaml diff --git a/bin/k8s/templates/postgresql-init-script-config.yaml b/bin/k8s/templates/common/postgresql/postgresql-init-script-config.yaml similarity index 100% rename from bin/k8s/templates/postgresql-init-script-config.yaml rename to bin/k8s/templates/common/postgresql/postgresql-init-script-config.yaml diff --git a/bin/k8s/templates/postgresql-persistence.yaml b/bin/k8s/templates/common/postgresql/postgresql-persistence.yaml similarity index 100% rename from bin/k8s/templates/postgresql-persistence.yaml rename to bin/k8s/templates/common/postgresql/postgresql-persistence.yaml diff --git a/bin/k8s/templates/pylsp.yaml b/bin/k8s/templates/common/pylsp/pylsp.yaml similarity index 100% rename from bin/k8s/templates/pylsp.yaml rename to bin/k8s/templates/common/pylsp/pylsp.yaml diff --git a/bin/k8s/templates/shared-editing-server.yaml b/bin/k8s/templates/common/shared-editing-server/shared-editing-server.yaml similarity index 100% rename from bin/k8s/templates/shared-editing-server.yaml rename to bin/k8s/templates/common/shared-editing-server/shared-editing-server.yaml diff --git a/bin/k8s/templates/webserver-deployment.yaml b/bin/k8s/templates/common/webserver/webserver-deployment.yaml similarity index 100% rename from bin/k8s/templates/webserver-deployment.yaml rename to bin/k8s/templates/common/webserver/webserver-deployment.yaml diff --git a/bin/k8s/templates/webserver-service.yaml b/bin/k8s/templates/common/webserver/webserver-service.yaml similarity index 100% rename from bin/k8s/templates/webserver-service.yaml rename to bin/k8s/templates/common/webserver/webserver-service.yaml diff --git a/bin/k8s/templates/workflow-compiling-service-deployment.yaml b/bin/k8s/templates/common/workflow-compiling-service/workflow-compiling-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/workflow-compiling-service-deployment.yaml rename to bin/k8s/templates/common/workflow-compiling-service/workflow-compiling-service-deployment.yaml diff --git a/bin/k8s/templates/workflow-compiling-service-service.yaml b/bin/k8s/templates/common/workflow-compiling-service/workflow-compiling-service-service.yaml similarity index 100% rename from bin/k8s/templates/workflow-compiling-service-service.yaml rename to bin/k8s/templates/common/workflow-compiling-service/workflow-compiling-service-service.yaml diff --git a/bin/k8s/templates/workflow-computing-unit-manager-deployment.yaml b/bin/k8s/templates/common/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml similarity index 100% rename from bin/k8s/templates/workflow-computing-unit-manager-deployment.yaml rename to bin/k8s/templates/common/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml diff --git a/bin/k8s/templates/workflow-computing-unit-manager-service-account.yaml b/bin/k8s/templates/common/workflow-computing-unit-manager/workflow-computing-unit-manager-service-account.yaml similarity index 100% rename from bin/k8s/templates/workflow-computing-unit-manager-service-account.yaml rename to bin/k8s/templates/common/workflow-computing-unit-manager/workflow-computing-unit-manager-service-account.yaml diff --git a/bin/k8s/templates/workflow-computing-unit-manager-service.yaml b/bin/k8s/templates/common/workflow-computing-unit-manager/workflow-computing-unit-manager-service.yaml similarity index 100% rename from bin/k8s/templates/workflow-computing-unit-manager-service.yaml rename to bin/k8s/templates/common/workflow-computing-unit-manager/workflow-computing-unit-manager-service.yaml diff --git a/bin/k8s/templates/workflow-computing-unit-master-prepull-daemonset.yaml b/bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-unit-master-prepull-daemonset.yaml similarity index 100% rename from bin/k8s/templates/workflow-computing-unit-master-prepull-daemonset.yaml rename to bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-unit-master-prepull-daemonset.yaml diff --git a/bin/k8s/templates/workflow-computing-unit-resource-quota.yaml b/bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-unit-resource-quota.yaml similarity index 100% rename from bin/k8s/templates/workflow-computing-unit-resource-quota.yaml rename to bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-unit-resource-quota.yaml diff --git a/bin/k8s/templates/workflow-computing-units-namespace.yaml b/bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-units-namespace.yaml similarity index 100% rename from bin/k8s/templates/workflow-computing-units-namespace.yaml rename to bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-units-namespace.yaml diff --git a/bin/k8s/templates/workflow-computing-units-service.yaml b/bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-units-service.yaml similarity index 100% rename from bin/k8s/templates/workflow-computing-units-service.yaml rename to bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-units-service.yaml diff --git a/bin/k8s/templates/minio-persistence.yaml b/bin/k8s/templates/onprem/minio-persistence.yaml similarity index 100% rename from bin/k8s/templates/minio-persistence.yaml rename to bin/k8s/templates/onprem/minio-persistence.yaml diff --git a/bin/k8s/values-development.yaml b/bin/k8s/values-development.yaml index dc7078e4688..aaed5e63374 100644 --- a/bin/k8s/values-development.yaml +++ b/bin/k8s/values-development.yaml @@ -18,8 +18,8 @@ texera: # Container image registry and tag for all Texera services # Override these to use a different registry or version - imageRegistry: ghcr.io/apache - imageTag: latest + imageRegistry: docker.io/apache + imageTag: 1.3.0-incubating-SNAPSHOT global: # Required by Bitnami sub-charts (postgresql, minio) to allow custom images @@ -352,6 +352,9 @@ texeraEnvVars: value: "" - name: USER_SYS_DOMAIN value: "" + - name: AUTH_JWT_SECRET + # Development-only default (256-bit HS256 secret). Production environments MUST override this with a different, securely generated secret. + value: "a7f3c8e9b14d2e6f5a0b9c3d8e1f4a6b2c5d7e9f0a3b6c8d1e4f7a9b2c5d8e1f" yWebsocketServer: name: y-websocket-server From ca0a4c7219aa4b5fd7819543dbb74fcdaaff65d7 Mon Sep 17 00:00:00 2001 From: ali Date: Tue, 23 Jun 2026 09:20:46 -0700 Subject: [PATCH 2/2] =?UTF-8?q?refactor(k8s):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20rename=20to=20base/on-prem,=20revert=20values-devel?= =?UTF-8?q?opment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename templates/common -> templates/base and templates/onprem -> templates/on-prem per review (@bobbai00), and update templates/README.md accordingly. - Revert values-development.yaml to apache/main: drop the imageRegistry/imageTag override (@Ma77Ball, @bobbai00) and the hardcoded AUTH_JWT_SECRET (@Ma77Ball), keeping this PR a pure mechanical reorg. Co-Authored-By: Claude Opus 4.8 (1M context) --- bin/k8s/templates/README.md | 14 +++++++------- .../access-control-service-deployment.yaml | 0 .../access-control-service-service.yaml | 0 .../agent-service-backend-traffic-policy.yaml | 0 .../agent-service/agent-service-deployment.yaml | 0 .../agent-service/agent-service-secret.yaml | 0 .../agent-service/agent-service-service.yaml | 0 .../config-service/config-service-deployment.yaml | 0 .../config-service/config-service-service.yaml | 0 .../example-data-loader-job.yaml | 0 .../external-names/external-names.yaml | 0 .../file-service/file-service-deployment.yaml | 0 .../file-service/file-service-service.yaml | 0 .../{common => base}/gateway/gateway-backend.yaml | 0 .../{common => base}/gateway/gateway-routes.yaml | 0 .../gateway/gateway-security-policy.yaml | 0 .../{common => base}/gateway/gateway.yaml | 0 .../{common => base}/lakefs/lakefs-secret.yaml | 0 .../{common => base}/lakefs/lakefs-setup-job.yaml | 0 .../lakekeeper/lakekeeper-init-job.yaml | 0 .../{common => base}/litellm/litellm-config.yaml | 0 .../litellm/litellm-deployment.yaml | 0 .../{common => base}/litellm/litellm-service.yaml | 0 .../postgresql/postgresql-init-script-config.yaml | 0 .../postgresql/postgresql-persistence.yaml | 0 .../templates/{common => base}/pylsp/pylsp.yaml | 0 .../shared-editing-server.yaml | 0 .../webserver/webserver-deployment.yaml | 0 .../webserver/webserver-service.yaml | 0 .../workflow-compiling-service-deployment.yaml | 0 .../workflow-compiling-service-service.yaml | 0 ...workflow-computing-unit-manager-deployment.yaml | 0 ...low-computing-unit-manager-service-account.yaml | 0 .../workflow-computing-unit-manager-service.yaml | 0 ...ow-computing-unit-master-prepull-daemonset.yaml | 0 .../workflow-computing-unit-resource-quota.yaml | 0 .../workflow-computing-units-namespace.yaml | 0 .../workflow-computing-units-service.yaml | 0 .../{onprem => on-prem}/minio-persistence.yaml | 0 bin/k8s/values-development.yaml | 7 ++----- 40 files changed, 9 insertions(+), 12 deletions(-) rename bin/k8s/templates/{common => base}/access-control-service/access-control-service-deployment.yaml (100%) rename bin/k8s/templates/{common => base}/access-control-service/access-control-service-service.yaml (100%) rename bin/k8s/templates/{common => base}/agent-service/agent-service-backend-traffic-policy.yaml (100%) rename bin/k8s/templates/{common => base}/agent-service/agent-service-deployment.yaml (100%) rename bin/k8s/templates/{common => base}/agent-service/agent-service-secret.yaml (100%) rename bin/k8s/templates/{common => base}/agent-service/agent-service-service.yaml (100%) rename bin/k8s/templates/{common => base}/config-service/config-service-deployment.yaml (100%) rename bin/k8s/templates/{common => base}/config-service/config-service-service.yaml (100%) rename bin/k8s/templates/{common => base}/example-data-loader/example-data-loader-job.yaml (100%) rename bin/k8s/templates/{common => base}/external-names/external-names.yaml (100%) rename bin/k8s/templates/{common => base}/file-service/file-service-deployment.yaml (100%) rename bin/k8s/templates/{common => base}/file-service/file-service-service.yaml (100%) rename bin/k8s/templates/{common => base}/gateway/gateway-backend.yaml (100%) rename bin/k8s/templates/{common => base}/gateway/gateway-routes.yaml (100%) rename bin/k8s/templates/{common => base}/gateway/gateway-security-policy.yaml (100%) rename bin/k8s/templates/{common => base}/gateway/gateway.yaml (100%) rename bin/k8s/templates/{common => base}/lakefs/lakefs-secret.yaml (100%) rename bin/k8s/templates/{common => base}/lakefs/lakefs-setup-job.yaml (100%) rename bin/k8s/templates/{common => base}/lakekeeper/lakekeeper-init-job.yaml (100%) rename bin/k8s/templates/{common => base}/litellm/litellm-config.yaml (100%) rename bin/k8s/templates/{common => base}/litellm/litellm-deployment.yaml (100%) rename bin/k8s/templates/{common => base}/litellm/litellm-service.yaml (100%) rename bin/k8s/templates/{common => base}/postgresql/postgresql-init-script-config.yaml (100%) rename bin/k8s/templates/{common => base}/postgresql/postgresql-persistence.yaml (100%) rename bin/k8s/templates/{common => base}/pylsp/pylsp.yaml (100%) rename bin/k8s/templates/{common => base}/shared-editing-server/shared-editing-server.yaml (100%) rename bin/k8s/templates/{common => base}/webserver/webserver-deployment.yaml (100%) rename bin/k8s/templates/{common => base}/webserver/webserver-service.yaml (100%) rename bin/k8s/templates/{common => base}/workflow-compiling-service/workflow-compiling-service-deployment.yaml (100%) rename bin/k8s/templates/{common => base}/workflow-compiling-service/workflow-compiling-service-service.yaml (100%) rename bin/k8s/templates/{common => base}/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml (100%) rename bin/k8s/templates/{common => base}/workflow-computing-unit-manager/workflow-computing-unit-manager-service-account.yaml (100%) rename bin/k8s/templates/{common => base}/workflow-computing-unit-manager/workflow-computing-unit-manager-service.yaml (100%) rename bin/k8s/templates/{common => base}/workflow-computing-unit-pool/workflow-computing-unit-master-prepull-daemonset.yaml (100%) rename bin/k8s/templates/{common => base}/workflow-computing-unit-pool/workflow-computing-unit-resource-quota.yaml (100%) rename bin/k8s/templates/{common => base}/workflow-computing-unit-pool/workflow-computing-units-namespace.yaml (100%) rename bin/k8s/templates/{common => base}/workflow-computing-unit-pool/workflow-computing-units-service.yaml (100%) rename bin/k8s/templates/{onprem => on-prem}/minio-persistence.yaml (100%) diff --git a/bin/k8s/templates/README.md b/bin/k8s/templates/README.md index bdf12287de4..7047a885d17 100644 --- a/bin/k8s/templates/README.md +++ b/bin/k8s/templates/README.md @@ -26,16 +26,16 @@ these subdirectories are purely organizational — they do not change rendering. | Folder | Contains | Renders when | |--------|----------|--------------| -| `common/` | Resources every deployment needs: the Texera micro-service Deployments/Services, the Envoy Gateway + routes, Postgres/LakeFS/Lakekeeper wiring, the computing-unit pool, RBAC and namespaces. | Always. | -| `onprem/` | Resources only used by a self-hosted / local deployment, e.g. the in-cluster MinIO persistence. | Gated on the relevant on-prem value (e.g. `minio.enabled`). | +| `base/` | Resources every deployment needs: the Texera micro-service Deployments/Services, the Envoy Gateway + routes, Postgres/LakeFS/Lakekeeper wiring, the computing-unit pool, RBAC and namespaces. | Always. | +| `on-prem/` | Resources only used by a self-hosted / local deployment, e.g. the in-cluster MinIO persistence. | Gated on the relevant on-prem value (e.g. `minio.enabled`). | | `aws/` | Resources only used on AWS/EKS, e.g. the external-S3 credentials Secret, the AWS NLB/EIP `EnvoyProxy`, and the autoscaler warm-pool placeholder. | Gated so they render to nothing off AWS (empty by default). | -Within `common/`, templates are further grouped into one subfolder per +Within `base/`, templates are further grouped into one subfolder per component, named after the service it belongs to, so every manifest for a given piece sits together: ``` -common/ +base/ access-control-service/ # access-control-service Deployment + Service agent-service/ # agent-service Deployment + Service + Secret + traffic policy config-service/ @@ -59,10 +59,10 @@ This nesting is also purely organizational — Helm still renders every file recursively. Guidelines for adding a template: -- Default to `common/`, in the subfolder for the component it belongs to (create +- Default to `base/`, in the subfolder for the component it belongs to (create a new one if it is a new component). Most resources are shared; only move a - file out to `aws/`/`onprem/` when it is genuinely specific to one hosting + file out to `aws/`/`on-prem/` when it is genuinely specific to one hosting environment. -- Anything under `aws/` or `onprem/` **must** be guarded by an `{{- if ... }}` +- Anything under `aws/` or `on-prem/` **must** be guarded by an `{{- if ... }}` on an opt-in value so that the default (on-prem) install is unaffected and an AWS install does not pick up on-prem-only resources. diff --git a/bin/k8s/templates/common/access-control-service/access-control-service-deployment.yaml b/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/common/access-control-service/access-control-service-deployment.yaml rename to bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml diff --git a/bin/k8s/templates/common/access-control-service/access-control-service-service.yaml b/bin/k8s/templates/base/access-control-service/access-control-service-service.yaml similarity index 100% rename from bin/k8s/templates/common/access-control-service/access-control-service-service.yaml rename to bin/k8s/templates/base/access-control-service/access-control-service-service.yaml diff --git a/bin/k8s/templates/common/agent-service/agent-service-backend-traffic-policy.yaml b/bin/k8s/templates/base/agent-service/agent-service-backend-traffic-policy.yaml similarity index 100% rename from bin/k8s/templates/common/agent-service/agent-service-backend-traffic-policy.yaml rename to bin/k8s/templates/base/agent-service/agent-service-backend-traffic-policy.yaml diff --git a/bin/k8s/templates/common/agent-service/agent-service-deployment.yaml b/bin/k8s/templates/base/agent-service/agent-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/common/agent-service/agent-service-deployment.yaml rename to bin/k8s/templates/base/agent-service/agent-service-deployment.yaml diff --git a/bin/k8s/templates/common/agent-service/agent-service-secret.yaml b/bin/k8s/templates/base/agent-service/agent-service-secret.yaml similarity index 100% rename from bin/k8s/templates/common/agent-service/agent-service-secret.yaml rename to bin/k8s/templates/base/agent-service/agent-service-secret.yaml diff --git a/bin/k8s/templates/common/agent-service/agent-service-service.yaml b/bin/k8s/templates/base/agent-service/agent-service-service.yaml similarity index 100% rename from bin/k8s/templates/common/agent-service/agent-service-service.yaml rename to bin/k8s/templates/base/agent-service/agent-service-service.yaml diff --git a/bin/k8s/templates/common/config-service/config-service-deployment.yaml b/bin/k8s/templates/base/config-service/config-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/common/config-service/config-service-deployment.yaml rename to bin/k8s/templates/base/config-service/config-service-deployment.yaml diff --git a/bin/k8s/templates/common/config-service/config-service-service.yaml b/bin/k8s/templates/base/config-service/config-service-service.yaml similarity index 100% rename from bin/k8s/templates/common/config-service/config-service-service.yaml rename to bin/k8s/templates/base/config-service/config-service-service.yaml diff --git a/bin/k8s/templates/common/example-data-loader/example-data-loader-job.yaml b/bin/k8s/templates/base/example-data-loader/example-data-loader-job.yaml similarity index 100% rename from bin/k8s/templates/common/example-data-loader/example-data-loader-job.yaml rename to bin/k8s/templates/base/example-data-loader/example-data-loader-job.yaml diff --git a/bin/k8s/templates/common/external-names/external-names.yaml b/bin/k8s/templates/base/external-names/external-names.yaml similarity index 100% rename from bin/k8s/templates/common/external-names/external-names.yaml rename to bin/k8s/templates/base/external-names/external-names.yaml diff --git a/bin/k8s/templates/common/file-service/file-service-deployment.yaml b/bin/k8s/templates/base/file-service/file-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/common/file-service/file-service-deployment.yaml rename to bin/k8s/templates/base/file-service/file-service-deployment.yaml diff --git a/bin/k8s/templates/common/file-service/file-service-service.yaml b/bin/k8s/templates/base/file-service/file-service-service.yaml similarity index 100% rename from bin/k8s/templates/common/file-service/file-service-service.yaml rename to bin/k8s/templates/base/file-service/file-service-service.yaml diff --git a/bin/k8s/templates/common/gateway/gateway-backend.yaml b/bin/k8s/templates/base/gateway/gateway-backend.yaml similarity index 100% rename from bin/k8s/templates/common/gateway/gateway-backend.yaml rename to bin/k8s/templates/base/gateway/gateway-backend.yaml diff --git a/bin/k8s/templates/common/gateway/gateway-routes.yaml b/bin/k8s/templates/base/gateway/gateway-routes.yaml similarity index 100% rename from bin/k8s/templates/common/gateway/gateway-routes.yaml rename to bin/k8s/templates/base/gateway/gateway-routes.yaml diff --git a/bin/k8s/templates/common/gateway/gateway-security-policy.yaml b/bin/k8s/templates/base/gateway/gateway-security-policy.yaml similarity index 100% rename from bin/k8s/templates/common/gateway/gateway-security-policy.yaml rename to bin/k8s/templates/base/gateway/gateway-security-policy.yaml diff --git a/bin/k8s/templates/common/gateway/gateway.yaml b/bin/k8s/templates/base/gateway/gateway.yaml similarity index 100% rename from bin/k8s/templates/common/gateway/gateway.yaml rename to bin/k8s/templates/base/gateway/gateway.yaml diff --git a/bin/k8s/templates/common/lakefs/lakefs-secret.yaml b/bin/k8s/templates/base/lakefs/lakefs-secret.yaml similarity index 100% rename from bin/k8s/templates/common/lakefs/lakefs-secret.yaml rename to bin/k8s/templates/base/lakefs/lakefs-secret.yaml diff --git a/bin/k8s/templates/common/lakefs/lakefs-setup-job.yaml b/bin/k8s/templates/base/lakefs/lakefs-setup-job.yaml similarity index 100% rename from bin/k8s/templates/common/lakefs/lakefs-setup-job.yaml rename to bin/k8s/templates/base/lakefs/lakefs-setup-job.yaml diff --git a/bin/k8s/templates/common/lakekeeper/lakekeeper-init-job.yaml b/bin/k8s/templates/base/lakekeeper/lakekeeper-init-job.yaml similarity index 100% rename from bin/k8s/templates/common/lakekeeper/lakekeeper-init-job.yaml rename to bin/k8s/templates/base/lakekeeper/lakekeeper-init-job.yaml diff --git a/bin/k8s/templates/common/litellm/litellm-config.yaml b/bin/k8s/templates/base/litellm/litellm-config.yaml similarity index 100% rename from bin/k8s/templates/common/litellm/litellm-config.yaml rename to bin/k8s/templates/base/litellm/litellm-config.yaml diff --git a/bin/k8s/templates/common/litellm/litellm-deployment.yaml b/bin/k8s/templates/base/litellm/litellm-deployment.yaml similarity index 100% rename from bin/k8s/templates/common/litellm/litellm-deployment.yaml rename to bin/k8s/templates/base/litellm/litellm-deployment.yaml diff --git a/bin/k8s/templates/common/litellm/litellm-service.yaml b/bin/k8s/templates/base/litellm/litellm-service.yaml similarity index 100% rename from bin/k8s/templates/common/litellm/litellm-service.yaml rename to bin/k8s/templates/base/litellm/litellm-service.yaml diff --git a/bin/k8s/templates/common/postgresql/postgresql-init-script-config.yaml b/bin/k8s/templates/base/postgresql/postgresql-init-script-config.yaml similarity index 100% rename from bin/k8s/templates/common/postgresql/postgresql-init-script-config.yaml rename to bin/k8s/templates/base/postgresql/postgresql-init-script-config.yaml diff --git a/bin/k8s/templates/common/postgresql/postgresql-persistence.yaml b/bin/k8s/templates/base/postgresql/postgresql-persistence.yaml similarity index 100% rename from bin/k8s/templates/common/postgresql/postgresql-persistence.yaml rename to bin/k8s/templates/base/postgresql/postgresql-persistence.yaml diff --git a/bin/k8s/templates/common/pylsp/pylsp.yaml b/bin/k8s/templates/base/pylsp/pylsp.yaml similarity index 100% rename from bin/k8s/templates/common/pylsp/pylsp.yaml rename to bin/k8s/templates/base/pylsp/pylsp.yaml diff --git a/bin/k8s/templates/common/shared-editing-server/shared-editing-server.yaml b/bin/k8s/templates/base/shared-editing-server/shared-editing-server.yaml similarity index 100% rename from bin/k8s/templates/common/shared-editing-server/shared-editing-server.yaml rename to bin/k8s/templates/base/shared-editing-server/shared-editing-server.yaml diff --git a/bin/k8s/templates/common/webserver/webserver-deployment.yaml b/bin/k8s/templates/base/webserver/webserver-deployment.yaml similarity index 100% rename from bin/k8s/templates/common/webserver/webserver-deployment.yaml rename to bin/k8s/templates/base/webserver/webserver-deployment.yaml diff --git a/bin/k8s/templates/common/webserver/webserver-service.yaml b/bin/k8s/templates/base/webserver/webserver-service.yaml similarity index 100% rename from bin/k8s/templates/common/webserver/webserver-service.yaml rename to bin/k8s/templates/base/webserver/webserver-service.yaml diff --git a/bin/k8s/templates/common/workflow-compiling-service/workflow-compiling-service-deployment.yaml b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/common/workflow-compiling-service/workflow-compiling-service-deployment.yaml rename to bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml diff --git a/bin/k8s/templates/common/workflow-compiling-service/workflow-compiling-service-service.yaml b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-service.yaml similarity index 100% rename from bin/k8s/templates/common/workflow-compiling-service/workflow-compiling-service-service.yaml rename to bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-service.yaml diff --git a/bin/k8s/templates/common/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml similarity index 100% rename from bin/k8s/templates/common/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml rename to bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml diff --git a/bin/k8s/templates/common/workflow-computing-unit-manager/workflow-computing-unit-manager-service-account.yaml b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-service-account.yaml similarity index 100% rename from bin/k8s/templates/common/workflow-computing-unit-manager/workflow-computing-unit-manager-service-account.yaml rename to bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-service-account.yaml diff --git a/bin/k8s/templates/common/workflow-computing-unit-manager/workflow-computing-unit-manager-service.yaml b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-service.yaml similarity index 100% rename from bin/k8s/templates/common/workflow-computing-unit-manager/workflow-computing-unit-manager-service.yaml rename to bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-service.yaml diff --git a/bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-unit-master-prepull-daemonset.yaml b/bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-unit-master-prepull-daemonset.yaml similarity index 100% rename from bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-unit-master-prepull-daemonset.yaml rename to bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-unit-master-prepull-daemonset.yaml diff --git a/bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-unit-resource-quota.yaml b/bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-unit-resource-quota.yaml similarity index 100% rename from bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-unit-resource-quota.yaml rename to bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-unit-resource-quota.yaml diff --git a/bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-units-namespace.yaml b/bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-units-namespace.yaml similarity index 100% rename from bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-units-namespace.yaml rename to bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-units-namespace.yaml diff --git a/bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-units-service.yaml b/bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-units-service.yaml similarity index 100% rename from bin/k8s/templates/common/workflow-computing-unit-pool/workflow-computing-units-service.yaml rename to bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-units-service.yaml diff --git a/bin/k8s/templates/onprem/minio-persistence.yaml b/bin/k8s/templates/on-prem/minio-persistence.yaml similarity index 100% rename from bin/k8s/templates/onprem/minio-persistence.yaml rename to bin/k8s/templates/on-prem/minio-persistence.yaml diff --git a/bin/k8s/values-development.yaml b/bin/k8s/values-development.yaml index aaed5e63374..dc7078e4688 100644 --- a/bin/k8s/values-development.yaml +++ b/bin/k8s/values-development.yaml @@ -18,8 +18,8 @@ texera: # Container image registry and tag for all Texera services # Override these to use a different registry or version - imageRegistry: docker.io/apache - imageTag: 1.3.0-incubating-SNAPSHOT + imageRegistry: ghcr.io/apache + imageTag: latest global: # Required by Bitnami sub-charts (postgresql, minio) to allow custom images @@ -352,9 +352,6 @@ texeraEnvVars: value: "" - name: USER_SYS_DOMAIN value: "" - - name: AUTH_JWT_SECRET - # Development-only default (256-bit HS256 secret). Production environments MUST override this with a different, securely generated secret. - value: "a7f3c8e9b14d2e6f5a0b9c3d8e1f4a6b2c5d7e9f0a3b6c8d1e4f7a9b2c5d8e1f" yWebsocketServer: name: y-websocket-server