diff --git a/bin/k8s/.helmignore b/bin/k8s/.helmignore new file mode 100644 index 00000000000..9577bcfcd70 --- /dev/null +++ b/bin/k8s/.helmignore @@ -0,0 +1,17 @@ +# Patterns to ignore when building Helm packages. +# Operating system / editor files +.DS_Store +*.swp +*.bak +*.tmp +*.orig +*~ +# Version control directories +.git/ +.gitignore +# Documentation kept alongside the chart for humans, not rendered as manifests. +# Markdown under templates/ (e.g. the common/aws/onprem layout guide) must be +# ignored so Helm does not try to load it as a Kubernetes manifest. +*.md +# git placeholders must not be loaded by Helm as manifests +.gitkeep diff --git a/bin/k8s/templates/README.md b/bin/k8s/templates/README.md new file mode 100644 index 00000000000..7047a885d17 --- /dev/null +++ b/bin/k8s/templates/README.md @@ -0,0 +1,68 @@ + + +# Helm template layout + +Templates are grouped by **where they apply**, so a reader can tell at a glance +which resources are deployment-agnostic and which are tied to a particular +hosting environment. Helm renders every file under `templates/` recursively, so +these subdirectories are purely organizational — they do not change rendering. + +| Folder | Contains | Renders when | +|--------|----------|--------------| +| `base/` | Resources every deployment needs: the Texera micro-service Deployments/Services, the Envoy Gateway + routes, Postgres/LakeFS/Lakekeeper wiring, the computing-unit pool, RBAC and namespaces. | Always. | +| `on-prem/` | Resources only used by a self-hosted / local deployment, e.g. the in-cluster MinIO persistence. | Gated on the relevant on-prem value (e.g. `minio.enabled`). | +| `aws/` | Resources only used on AWS/EKS, e.g. the external-S3 credentials Secret, the AWS NLB/EIP `EnvoyProxy`, and the autoscaler warm-pool placeholder. | Gated so they render to nothing off AWS (empty by default). | + +Within `base/`, templates are further grouped into one subfolder per +component, named after the service it belongs to, so every manifest for a given +piece sits together: + +``` +base/ + access-control-service/ # access-control-service Deployment + Service + agent-service/ # agent-service Deployment + Service + Secret + traffic policy + config-service/ + file-service/ + gateway/ # Envoy Gateway + routes + backends + security policy + lakefs/ + lakekeeper/ + litellm/ + postgresql/ # in-cluster Postgres PV/PVC + init scripts + webserver/ # dashboard / webserver Deployment + Service + workflow-compiling-service/ + workflow-computing-unit-manager/ # the CU manager service + RBAC + workflow-computing-unit-pool/ # CU pool namespace, quota, prepull, service + example-data-loader/ + external-names/ # ExternalName service aliases + pylsp/ # python language server + shared-editing-server/ # y-websocket collaborative editing +``` + +This nesting is also purely organizational — Helm still renders every file +recursively. + +Guidelines for adding a template: +- Default to `base/`, in the subfolder for the component it belongs to (create + a new one if it is a new component). Most resources are shared; only move a + file out to `aws/`/`on-prem/` when it is genuinely specific to one hosting + environment. +- Anything under `aws/` or `on-prem/` **must** be guarded by an `{{- if ... }}` + on an opt-in value so that the default (on-prem) install is unaffected and an + AWS install does not pick up on-prem-only resources. diff --git a/bin/k8s/templates/aws/.gitkeep b/bin/k8s/templates/aws/.gitkeep new file mode 100644 index 00000000000..87fae5f6485 --- /dev/null +++ b/bin/k8s/templates/aws/.gitkeep @@ -0,0 +1,2 @@ +# Placeholder so the (initially empty) aws/ template folder is tracked in git. +# AWS-only, value-gated templates are added by later PRs in the AWS/EKS series. diff --git a/bin/k8s/templates/access-control-service-deployment.yaml b/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/access-control-service-deployment.yaml rename to bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml diff --git a/bin/k8s/templates/access-control-service-service.yaml b/bin/k8s/templates/base/access-control-service/access-control-service-service.yaml similarity index 100% rename from bin/k8s/templates/access-control-service-service.yaml rename to bin/k8s/templates/base/access-control-service/access-control-service-service.yaml diff --git a/bin/k8s/templates/agent-service-backend-traffic-policy.yaml b/bin/k8s/templates/base/agent-service/agent-service-backend-traffic-policy.yaml similarity index 100% rename from bin/k8s/templates/agent-service-backend-traffic-policy.yaml rename to bin/k8s/templates/base/agent-service/agent-service-backend-traffic-policy.yaml diff --git a/bin/k8s/templates/agent-service-deployment.yaml b/bin/k8s/templates/base/agent-service/agent-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/agent-service-deployment.yaml rename to bin/k8s/templates/base/agent-service/agent-service-deployment.yaml diff --git a/bin/k8s/templates/agent-service-secret.yaml b/bin/k8s/templates/base/agent-service/agent-service-secret.yaml similarity index 100% rename from bin/k8s/templates/agent-service-secret.yaml rename to bin/k8s/templates/base/agent-service/agent-service-secret.yaml diff --git a/bin/k8s/templates/agent-service-service.yaml b/bin/k8s/templates/base/agent-service/agent-service-service.yaml similarity index 100% rename from bin/k8s/templates/agent-service-service.yaml rename to bin/k8s/templates/base/agent-service/agent-service-service.yaml diff --git a/bin/k8s/templates/config-service-deployment.yaml b/bin/k8s/templates/base/config-service/config-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/config-service-deployment.yaml rename to bin/k8s/templates/base/config-service/config-service-deployment.yaml diff --git a/bin/k8s/templates/config-service-service.yaml b/bin/k8s/templates/base/config-service/config-service-service.yaml similarity index 100% rename from bin/k8s/templates/config-service-service.yaml rename to bin/k8s/templates/base/config-service/config-service-service.yaml diff --git a/bin/k8s/templates/example-data-loader-job.yaml b/bin/k8s/templates/base/example-data-loader/example-data-loader-job.yaml similarity index 100% rename from bin/k8s/templates/example-data-loader-job.yaml rename to bin/k8s/templates/base/example-data-loader/example-data-loader-job.yaml diff --git a/bin/k8s/templates/external-names.yaml b/bin/k8s/templates/base/external-names/external-names.yaml similarity index 100% rename from bin/k8s/templates/external-names.yaml rename to bin/k8s/templates/base/external-names/external-names.yaml diff --git a/bin/k8s/templates/file-service-deployment.yaml b/bin/k8s/templates/base/file-service/file-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/file-service-deployment.yaml rename to bin/k8s/templates/base/file-service/file-service-deployment.yaml diff --git a/bin/k8s/templates/file-service-service.yaml b/bin/k8s/templates/base/file-service/file-service-service.yaml similarity index 100% rename from bin/k8s/templates/file-service-service.yaml rename to bin/k8s/templates/base/file-service/file-service-service.yaml diff --git a/bin/k8s/templates/gateway-backend.yaml b/bin/k8s/templates/base/gateway/gateway-backend.yaml similarity index 100% rename from bin/k8s/templates/gateway-backend.yaml rename to bin/k8s/templates/base/gateway/gateway-backend.yaml diff --git a/bin/k8s/templates/gateway-routes.yaml b/bin/k8s/templates/base/gateway/gateway-routes.yaml similarity index 100% rename from bin/k8s/templates/gateway-routes.yaml rename to bin/k8s/templates/base/gateway/gateway-routes.yaml diff --git a/bin/k8s/templates/gateway-security-policy.yaml b/bin/k8s/templates/base/gateway/gateway-security-policy.yaml similarity index 100% rename from bin/k8s/templates/gateway-security-policy.yaml rename to bin/k8s/templates/base/gateway/gateway-security-policy.yaml diff --git a/bin/k8s/templates/gateway.yaml b/bin/k8s/templates/base/gateway/gateway.yaml similarity index 100% rename from bin/k8s/templates/gateway.yaml rename to bin/k8s/templates/base/gateway/gateway.yaml diff --git a/bin/k8s/templates/lakefs-secret.yaml b/bin/k8s/templates/base/lakefs/lakefs-secret.yaml similarity index 100% rename from bin/k8s/templates/lakefs-secret.yaml rename to bin/k8s/templates/base/lakefs/lakefs-secret.yaml diff --git a/bin/k8s/templates/lakefs-setup-job.yaml b/bin/k8s/templates/base/lakefs/lakefs-setup-job.yaml similarity index 100% rename from bin/k8s/templates/lakefs-setup-job.yaml rename to bin/k8s/templates/base/lakefs/lakefs-setup-job.yaml diff --git a/bin/k8s/templates/lakekeeper-init-job.yaml b/bin/k8s/templates/base/lakekeeper/lakekeeper-init-job.yaml similarity index 100% rename from bin/k8s/templates/lakekeeper-init-job.yaml rename to bin/k8s/templates/base/lakekeeper/lakekeeper-init-job.yaml diff --git a/bin/k8s/templates/litellm-config.yaml b/bin/k8s/templates/base/litellm/litellm-config.yaml similarity index 100% rename from bin/k8s/templates/litellm-config.yaml rename to bin/k8s/templates/base/litellm/litellm-config.yaml diff --git a/bin/k8s/templates/litellm-deployment.yaml b/bin/k8s/templates/base/litellm/litellm-deployment.yaml similarity index 100% rename from bin/k8s/templates/litellm-deployment.yaml rename to bin/k8s/templates/base/litellm/litellm-deployment.yaml diff --git a/bin/k8s/templates/litellm-service.yaml b/bin/k8s/templates/base/litellm/litellm-service.yaml similarity index 100% rename from bin/k8s/templates/litellm-service.yaml rename to bin/k8s/templates/base/litellm/litellm-service.yaml diff --git a/bin/k8s/templates/postgresql-init-script-config.yaml b/bin/k8s/templates/base/postgresql/postgresql-init-script-config.yaml similarity index 100% rename from bin/k8s/templates/postgresql-init-script-config.yaml rename to bin/k8s/templates/base/postgresql/postgresql-init-script-config.yaml diff --git a/bin/k8s/templates/postgresql-persistence.yaml b/bin/k8s/templates/base/postgresql/postgresql-persistence.yaml similarity index 100% rename from bin/k8s/templates/postgresql-persistence.yaml rename to bin/k8s/templates/base/postgresql/postgresql-persistence.yaml diff --git a/bin/k8s/templates/pylsp.yaml b/bin/k8s/templates/base/pylsp/pylsp.yaml similarity index 100% rename from bin/k8s/templates/pylsp.yaml rename to bin/k8s/templates/base/pylsp/pylsp.yaml diff --git a/bin/k8s/templates/shared-editing-server.yaml b/bin/k8s/templates/base/shared-editing-server/shared-editing-server.yaml similarity index 100% rename from bin/k8s/templates/shared-editing-server.yaml rename to bin/k8s/templates/base/shared-editing-server/shared-editing-server.yaml diff --git a/bin/k8s/templates/webserver-deployment.yaml b/bin/k8s/templates/base/webserver/webserver-deployment.yaml similarity index 100% rename from bin/k8s/templates/webserver-deployment.yaml rename to bin/k8s/templates/base/webserver/webserver-deployment.yaml diff --git a/bin/k8s/templates/webserver-service.yaml b/bin/k8s/templates/base/webserver/webserver-service.yaml similarity index 100% rename from bin/k8s/templates/webserver-service.yaml rename to bin/k8s/templates/base/webserver/webserver-service.yaml diff --git a/bin/k8s/templates/workflow-compiling-service-deployment.yaml b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml similarity index 100% rename from bin/k8s/templates/workflow-compiling-service-deployment.yaml rename to bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml diff --git a/bin/k8s/templates/workflow-compiling-service-service.yaml b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-service.yaml similarity index 100% rename from bin/k8s/templates/workflow-compiling-service-service.yaml rename to bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-service.yaml diff --git a/bin/k8s/templates/workflow-computing-unit-manager-deployment.yaml b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml similarity index 100% rename from bin/k8s/templates/workflow-computing-unit-manager-deployment.yaml rename to bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml diff --git a/bin/k8s/templates/workflow-computing-unit-manager-service-account.yaml b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-service-account.yaml similarity index 100% rename from bin/k8s/templates/workflow-computing-unit-manager-service-account.yaml rename to bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-service-account.yaml diff --git a/bin/k8s/templates/workflow-computing-unit-manager-service.yaml b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-service.yaml similarity index 100% rename from bin/k8s/templates/workflow-computing-unit-manager-service.yaml rename to bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-service.yaml diff --git a/bin/k8s/templates/workflow-computing-unit-master-prepull-daemonset.yaml b/bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-unit-master-prepull-daemonset.yaml similarity index 100% rename from bin/k8s/templates/workflow-computing-unit-master-prepull-daemonset.yaml rename to bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-unit-master-prepull-daemonset.yaml diff --git a/bin/k8s/templates/workflow-computing-unit-resource-quota.yaml b/bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-unit-resource-quota.yaml similarity index 100% rename from bin/k8s/templates/workflow-computing-unit-resource-quota.yaml rename to bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-unit-resource-quota.yaml diff --git a/bin/k8s/templates/workflow-computing-units-namespace.yaml b/bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-units-namespace.yaml similarity index 100% rename from bin/k8s/templates/workflow-computing-units-namespace.yaml rename to bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-units-namespace.yaml diff --git a/bin/k8s/templates/workflow-computing-units-service.yaml b/bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-units-service.yaml similarity index 100% rename from bin/k8s/templates/workflow-computing-units-service.yaml rename to bin/k8s/templates/base/workflow-computing-unit-pool/workflow-computing-units-service.yaml diff --git a/bin/k8s/templates/minio-persistence.yaml b/bin/k8s/templates/on-prem/minio-persistence.yaml similarity index 100% rename from bin/k8s/templates/minio-persistence.yaml rename to bin/k8s/templates/on-prem/minio-persistence.yaml