diff --git a/ROADMAP.md b/ROADMAP.md index a7459a5..53da4e5 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -215,10 +215,17 @@ prerequisite (Longhorn); the rest are convenience + robustness. merge/webhook today — B1–B3; PR-preview was deferred to Argo). Enhancement. ### J. App catalog / examples -- [ ] **J1 — example manifests** for the target apps under `examples/` - (Ollama + Open WebUI on GPU, Jellyfin on NFS, MinIO, Authentik-as-SSO, a - static blog behind the tunnel) — the deploy-today path, documented - end-to-end. Not a code gap; lowers the activation energy. +- [x] **J1 — example manifests** under `examples/homelab/` — the full stack + end-to-end: a GPU-capable k3s `Cluster` (`.cue`, `openctl validate`-checked + offline) with a GPU pool + a Longhorn-prepped storage pool (`nodePrep: + open-iscsi`); a `Platform` (traefik/cloudflared/longhorn/nfs/nvidia + a + generic `kube-prometheus-stack`); a Cloudflare `Tunnel` + `DNSRecord`s that + `$ref` its `cnameTarget`; and `HelmRelease`s for Ollama+Open WebUI (GPU), + Jellyfin (NFS), MinIO, Authentik (SSO). README walks the deploy-today path + and the three wiring mechanisms (`$ref` outputs, `$secret` credentials, + `nodePrep` prerequisites). Not a code gap; lowers activation energy. The + `.cue` cluster validates against the compiled-in schema; the plugin-kind + YAML is well-formed and schema-accurate. ## Architecture & consolidation (2026-07-13) diff --git a/examples/homelab/01-cluster.cue b/examples/homelab/01-cluster.cue new file mode 100644 index 0000000..58d3783 --- /dev/null +++ b/examples/homelab/01-cluster.cue @@ -0,0 +1,73 @@ +// Homelab k3s cluster: an HA-ish control plane plus two purpose-built worker +// pools — a GPU pool for local models (Ollama/Open WebUI) and a storage pool +// prepped for Longhorn (open-iscsi installed via nodePrep). +// +// Validate this file with: openctl validate -f examples/homelab/01-cluster.cue +// +// The cluster publishes status.outputs.kubeconfigPath, which the workload +// manifests (Platform, HelmReleases) $ref to target this cluster — that ref +// also DAG-orders them after the cluster is Ready. +import "openctl.io/schemas/k3s" + +k3s.#Cluster & { + metadata: name: "home" + spec: { + compute: { + provider: "proxmox" + // A cloud image is downloaded once into a template; nodes clone it. + image: { + url: "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img" + storage: "local" // where the downloaded image + snippets live + diskStorage: "local-lvm" // where node disks are allocated + } + default: {cpus: 2, memoryMB: 4096, diskGB: 40} + } + nodes: { + controlPlane: count: 1 + + workers: [ + // General-purpose workloads (blogs, MinIO, Authentik, Jellyfin). + {name: "general", count: 2, size: {cpus: 4, memoryMB: 8192, diskGB: 60}}, + + // GPU pool for a local model. Pinned to the host with the card; + // the pool's VMs get q35 + OVMF + the GPU passed through. Pair + // with the nvidiaDevicePlugin Platform component so workloads can + // request nvidia.com/gpu. + { + name: "gpu" + count: 1 + nodes: ["pve-gpu"] + size: {cpus: 8, memoryMB: 24576, diskGB: 80} + gpu: { + efiStorage: "local-lvm" + devices: [ + {mapping: "rtx4090", primaryGPU: true}, + ] + } + }, + + // Storage pool for Longhorn replicated block storage. nodePrep + // installs open-iscsi (Longhorn's node prerequisite) on first + // boot via cloud-init. + { + name: "storage" + count: 3 + size: {cpus: 4, memoryMB: 8192, diskGB: 200} + nodePrep: { + packages: ["open-iscsi"] + runcmd: ["systemctl enable --now iscsid"] + } + }, + ] + } + network: staticIPs: { + startIP: "192.168.1.100" + gateway: "192.168.1.1" + netmask: "24" + } + ssh: { + privateKeyPath: "~/.ssh/id_ed25519" + publicKeys: ["ssh-ed25519 AAAA... you@homelab"] + } + } +} diff --git a/examples/homelab/02-platform.yaml b/examples/homelab/02-platform.yaml new file mode 100644 index 0000000..4fca375 --- /dev/null +++ b/examples/homelab/02-platform.yaml @@ -0,0 +1,69 @@ +# Platform: the opt-in infra layer for the cluster — ingress, tunnel, storage +# classes, GPU scheduling. Each enabled component installs one Helm release. +# Nothing is enabled by default; this is the homelab's chosen set. +# +# The kubeconfigPath $ref targets the k3s Cluster from 01-cluster.cue: openctl +# resolves it to the on-disk kubeconfig path AND orders this apply after the +# cluster is Ready. +apiVersion: k8s.openctl.io/v1 +kind: Platform +metadata: + name: home +spec: + kubeconfigPath: + $ref: + apiVersion: k3s.openctl.io/v1 + kind: Cluster + name: home + field: status.outputs.kubeconfigPath + + # --- Named presets (default chart coords; enable with a flag) --- + + # Ingress controller. + traefik: + enabled: true + + # Cloudflare Tunnel connector. Its run token is pulled straight from the + # Tunnel's get-token action via the `action` $secret provider — no manual + # token copying, and only the marker is persisted (never the token). + cloudflared: + enabled: true + values: + cloudflare: + tunnel_token: + $secret: + provider: action + key: "cloudflare.openctl.io/v1/Tunnel/home#get-token" + + # Replicated block storage. The storage worker pool already has open-iscsi + # (see 01-cluster.cue nodePrep), which Longhorn requires. + longhorn: + enabled: true + values: + defaultSettings: + defaultReplicaCount: 2 + + # Dynamic NFS-backed StorageClass off the Synology (media, Home Assistant). + nfsProvisioner: + enabled: true + values: + nfs: + server: "192.168.1.10" + path: "/volume1/k8s" + + # Advertise nvidia.com/gpu on the GPU pool so Ollama can request one. + nvidiaDevicePlugin: + enabled: true + + # --- Generic components: any chart, no code change (K4a) --- + # These install exactly like the presets but aren't baked into openctl. + components: + kube-prometheus-stack: + chart: + repo: https://prometheus-community.github.io/helm-charts + name: kube-prometheus-stack + version: "65.0.0" + namespace: monitoring + values: + grafana: + enabled: true diff --git a/examples/homelab/03-tunnel.yaml b/examples/homelab/03-tunnel.yaml new file mode 100644 index 0000000..821603b --- /dev/null +++ b/examples/homelab/03-tunnel.yaml @@ -0,0 +1,23 @@ +# Cloudflare Tunnel: one connector fronting every public app. Ingress rules map +# public hostnames to in-cluster Traefik (or a service directly). cloudflared +# (installed by the Platform) runs the connector using this tunnel's token. +# +# A catch-all 404 is appended automatically after the last host-scoped rule. +apiVersion: cloudflare.openctl.io/v1 +kind: Tunnel +metadata: + name: home +spec: + # accountId optional if the provider config sets defaults.accountId. + ingress: + # Everything routes through Traefik, which does host-based routing in-cluster. + - hostname: "chat.example.com" + service: "https://traefik.traefik.svc.cluster.local:443" + - hostname: "jellyfin.example.com" + service: "https://traefik.traefik.svc.cluster.local:443" + - hostname: "minio.example.com" + service: "https://traefik.traefik.svc.cluster.local:443" + - hostname: "auth.example.com" + service: "https://traefik.traefik.svc.cluster.local:443" + - hostname: "blog.example.com" + service: "https://traefik.traefik.svc.cluster.local:443" diff --git a/examples/homelab/04-dns.yaml b/examples/homelab/04-dns.yaml new file mode 100644 index 0000000..c1226be --- /dev/null +++ b/examples/homelab/04-dns.yaml @@ -0,0 +1,50 @@ +# Public DNS: one CNAME per app, each pointing at the Tunnel. Instead of +# hand-copying ".cfargotunnel.com", the content $refs the Tunnel's +# status.cnameTarget — openctl resolves it at apply time (and orders each +# record after the Tunnel). Change the tunnel and the records follow. +# +# One document per record (YAML --- separated). +apiVersion: cloudflare.openctl.io/v1 +kind: DNSRecord +metadata: + name: chat +spec: + type: CNAME + name: "chat.example.com" + proxied: true + content: + $ref: + apiVersion: cloudflare.openctl.io/v1 + kind: Tunnel + name: home + field: status.cnameTarget +--- +apiVersion: cloudflare.openctl.io/v1 +kind: DNSRecord +metadata: + name: jellyfin +spec: + type: CNAME + name: "jellyfin.example.com" + proxied: true + content: + $ref: + apiVersion: cloudflare.openctl.io/v1 + kind: Tunnel + name: home + field: status.cnameTarget +--- +apiVersion: cloudflare.openctl.io/v1 +kind: DNSRecord +metadata: + name: blog +spec: + type: CNAME + name: "blog.example.com" + proxied: true + content: + $ref: + apiVersion: cloudflare.openctl.io/v1 + kind: Tunnel + name: home + field: status.cnameTarget diff --git a/examples/homelab/05-ollama-openwebui.yaml b/examples/homelab/05-ollama-openwebui.yaml new file mode 100644 index 0000000..84ce8d1 --- /dev/null +++ b/examples/homelab/05-ollama-openwebui.yaml @@ -0,0 +1,69 @@ +# Local model stack: Ollama (the model server, scheduled onto the GPU pool via +# nvidia.com/gpu) + Open WebUI (the chat front-end, exposed at chat.example.com +# through the Tunnel). Both are stock Helm charts — openctl just installs them +# onto the cluster the kubeconfigPath $ref points at. +apiVersion: k8s.openctl.io/v1 +kind: HelmRelease +metadata: + name: ollama +spec: + kubeconfigPath: + $ref: + apiVersion: k3s.openctl.io/v1 + kind: Cluster + name: home + field: status.outputs.kubeconfigPath + namespace: ai + createNamespace: true + chart: + repo: https://otwld.github.io/ollama-helm + name: ollama + version: "1.30.0" + values: + ollama: + gpu: + enabled: true + type: nvidia + number: 1 + models: + pull: + - llama3.2 + # Request the GPU advertised by the nvidiaDevicePlugin Platform component; + # nodeSelector/tolerations pin it to the GPU pool if you label those nodes. + resources: + limits: + nvidia.com/gpu: 1 + persistentVolume: + enabled: true + storageClass: longhorn + size: 50Gi +--- +apiVersion: k8s.openctl.io/v1 +kind: HelmRelease +metadata: + name: open-webui +spec: + kubeconfigPath: + $ref: + apiVersion: k3s.openctl.io/v1 + kind: Cluster + name: home + field: status.outputs.kubeconfigPath + namespace: ai + createNamespace: true + chart: + repo: https://helm.openwebui.com + name: open-webui + version: "5.4.0" + values: + # Point the UI at the in-cluster Ollama service instead of bundling one. + ollama: + enabled: false + ollamaUrls: + - http://ollama.ai.svc.cluster.local:11434 + persistence: + enabled: true + storageClass: longhorn + ingress: + enabled: true + host: chat.example.com diff --git a/examples/homelab/06-media-and-services.yaml b/examples/homelab/06-media-and-services.yaml new file mode 100644 index 0000000..8d60eb2 --- /dev/null +++ b/examples/homelab/06-media-and-services.yaml @@ -0,0 +1,94 @@ +# The rest of the self-hosted stack, each a stock Helm chart onto the same +# cluster: Jellyfin (media, on NFS), MinIO (S3-compatible object store, on +# Longhorn), and Authentik (the SSO IdP — also openctl's own OIDC provider via +# auth.oidc). Exposed through Traefik + the Tunnel (see 03/04). +apiVersion: k8s.openctl.io/v1 +kind: HelmRelease +metadata: + name: jellyfin +spec: + kubeconfigPath: + $ref: + apiVersion: k3s.openctl.io/v1 + kind: Cluster + name: home + field: status.outputs.kubeconfigPath + namespace: media + createNamespace: true + chart: + repo: https://jellyfin.github.io/jellyfin-helm + name: jellyfin + version: "2.3.0" + values: + persistence: + config: + storageClass: longhorn + size: 10Gi + # Media library lives on the NFS share (nfsProvisioner StorageClass). + media: + storageClass: nfs-client + size: 2Ti + ingress: + enabled: true + hosts: + - host: jellyfin.example.com + paths: [{path: "/"}] +--- +apiVersion: k8s.openctl.io/v1 +kind: HelmRelease +metadata: + name: minio +spec: + kubeconfigPath: + $ref: + apiVersion: k3s.openctl.io/v1 + kind: Cluster + name: home + field: status.outputs.kubeconfigPath + namespace: minio + createNamespace: true + chart: + repo: https://charts.min.io + name: minio + version: "5.4.0" + values: + mode: standalone + persistence: + enabled: true + storageClass: longhorn + size: 200Gi + # Prefer a $secret for the root credentials in a real deployment. + rootUser: admin + rootPassword: + $secret: + provider: vault + key: "secret/data/minio#rootPassword" +--- +apiVersion: k8s.openctl.io/v1 +kind: HelmRelease +metadata: + name: authentik +spec: + kubeconfigPath: + $ref: + apiVersion: k3s.openctl.io/v1 + kind: Cluster + name: home + field: status.outputs.kubeconfigPath + namespace: authentik + createNamespace: true + chart: + repo: https://charts.goauthentik.io + name: authentik + version: "2024.10.1" + values: + authentik: + secret_key: + $secret: + provider: vault + key: "secret/data/authentik#secretKey" + server: + ingress: + enabled: true + hosts: + - auth.example.com diff --git a/examples/homelab/README.md b/examples/homelab/README.md new file mode 100644 index 0000000..5e70776 --- /dev/null +++ b/examples/homelab/README.md @@ -0,0 +1,69 @@ +# Homelab: full stack, end to end + +A worked example of the whole homelab vision through openctl — a GPU-capable +k3s cluster, its infra layer, and the self-hosted apps — using only primitives +that ship today. Nothing here is bespoke: the cluster is a `k3s.Cluster`, every +app is a stock Helm chart wrapped in a `HelmRelease`, and public exposure is a +`Tunnel` + `DNSRecord`. + +## What gets built + +| File | Kind(s) | What | +|------|---------|------| +| `01-cluster.cue` | `k3s/Cluster` | Control plane + a **general** pool, a **GPU** pool (passthrough), and a **storage** pool prepped for Longhorn (`nodePrep: open-iscsi`) | +| `02-platform.yaml` | `k8s/Platform` | traefik, cloudflared, longhorn, nfsProvisioner, nvidiaDevicePlugin, and a generic `kube-prometheus-stack` | +| `03-tunnel.yaml` | `cloudflare/Tunnel` | One connector fronting every public hostname | +| `04-dns.yaml` | `cloudflare/DNSRecord` | CNAMEs that `$ref` the Tunnel's `status.cnameTarget` | +| `05-ollama-openwebui.yaml` | `k8s/HelmRelease` | Ollama on the GPU + Open WebUI at `chat.example.com` | +| `06-media-and-services.yaml` | `k8s/HelmRelease` | Jellyfin (NFS), MinIO (Longhorn), Authentik (SSO) | + +## The three wiring mechanisms + +Everything is glued together by three openctl features — no hand-copied values: + +1. **`$ref` for outputs.** Each workload's `kubeconfigPath` `$ref`s the cluster's + `status.outputs.kubeconfigPath`. openctl resolves it to the on-disk kubeconfig + *and* schedules the workload **after** the cluster is Ready. Same mechanism + points each `DNSRecord.content` at the `Tunnel.status.cnameTarget`. Run + `openctl explain ` to see what a kind exposes to `$ref`. +2. **`$secret` for credentials.** cloudflared's tunnel token comes from the + Tunnel's `get-token` action via the `action` secret provider; MinIO/Authentik + secrets come from Vault. Only the marker is ever persisted — never the value, + so nothing sensitive lands in git. +3. **Node prep for prerequisites.** The storage pool installs `open-iscsi` on + first boot (`nodePrep`), so Longhorn works with no manual node touch. + +## Apply order + +openctl derives ordering from the `$ref` edges, so you can apply the whole +directory and it sorts itself out — but conceptually: + +``` +01-cluster.cue # the k3s cluster (VMs → k3s → kubeconfig) +02-platform.yaml # ingress + storage classes + GPU scheduling +03-tunnel.yaml # create the Cloudflare tunnel (publishes cnameTarget) +04-dns.yaml # CNAMEs → tunnel (waits on 03 via $ref) +05/06 *.yaml # the apps (wait on 02 for storage/ingress) +``` + +```sh +openctl validate -f examples/homelab/01-cluster.cue # the cluster schema-checks offline +openctl apply -f examples/homelab/01-cluster.cue +openctl apply -f examples/homelab/ # the rest +``` + +## Prerequisites & placeholders + +- **Proxmox**: a `local-lvm` (disks) + a snippets-capable storage (`local`), + and a host `pve-gpu` with an NVIDIA card exposed as the `rtx4090` resource + mapping. Adjust `nodes:`/`gpu.devices` to your hardware. +- **Cloudflare**: a configured provider credential + zone; replace + `example.com` with your domain. +- **Vault** (optional): the MinIO/Authentik `$secret`s assume a Vault backend; + swap for `env`/`file` providers if you don't run one. +- Replace the SSH key, IP range (`192.168.1.x`), NFS `server`/`path`, and chart + versions with your own. Chart versions are pinned as examples — check for + current releases. + +These are the deploy-today primitives; the point is that the homelab is just +declarative infra + stock charts, wired by `$ref`/`$secret`.