diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1ff01350ae..02f96d86b7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -728,6 +728,35 @@ jobs: ${{ needs.prepare.outputs.image_registry }}/nvmetal-carbide@${{ needs.build-release-container-x86_64.outputs.image_digest }} \ ${{ needs.prepare.outputs.image_registry }}/nvmetal-carbide-aarch64@${{ needs.build-release-container-aarch64.outputs.image_digest }} + build-machine-a-tron: + if: >- + ${{ + always() + && github.event_name != 'schedule' + && needs.prepare.result == 'success' + && needs.prepare.outputs.source_files_changed == 'true' + }} + needs: + - prepare + uses: ./.github/workflows/docker-build.yml + with: + dockerfile_path: crates/machine-a-tron/Dockerfile + context_path: . + build_args: >- + ${{ format('{{"VERSION":"{0}","CI_COMMIT_SHORT_SHA":"{1}"}}', + needs.prepare.outputs.version, + needs.prepare.outputs.short_sha) }} + image_name: ${{ needs.prepare.outputs.image_registry }}/machine-a-tron + image_tag: ${{ needs.prepare.outputs.version }} + platforms: linux/amd64 + runner: linux-amd64-cpu16 + push: ${{ needs.prepare.outputs.publish_images == 'true' }} + load: false + scan: true + tag_latest: false + timeout_minutes: 60 + secrets: inherit + test-release-container-services: if: >- ${{ @@ -1716,6 +1745,7 @@ jobs: - build-artifacts-container-aarch64 - build-release-container-x86_64 - build-release-container-aarch64 + - build-machine-a-tron - test-release-container-services - build-boot-artifacts-x86 - build-boot-artifacts-bfb @@ -1816,6 +1846,7 @@ jobs: runs-on: linux-amd64-cpu4 needs: - build-release-container-x86_64 + - build-machine-a-tron - build-forge-cli-x86_64 - build-release-artifacts-x86-host - build-release-artifacts-arm-host @@ -1846,6 +1877,7 @@ jobs: - build-artifacts-container-aarch64 - build-release-container-x86_64 - build-release-container-aarch64 + - build-machine-a-tron - test-release-container-services - build-boot-artifacts-x86 - build-boot-artifacts-bfb diff --git a/crates/api-core/src/auth/internal_rbac_rules.rs b/crates/api-core/src/auth/internal_rbac_rules.rs index dcf4974919..4b1e11e0d6 100644 --- a/crates/api-core/src/auth/internal_rbac_rules.rs +++ b/crates/api-core/src/auth/internal_rbac_rules.rs @@ -322,7 +322,10 @@ impl InternalRBACRules { x.perm("UpdateInstancePhoneHomeLastContact", vec![Agent]); x.perm("SetHostUefiPassword", vec![ForgeAdminCLI]); x.perm("ClearHostUefiPassword", vec![ForgeAdminCLI]); - x.perm("AddExpectedMachine", vec![ForgeAdminCLI, SiteAgent, Flow]); + x.perm( + "AddExpectedMachine", + vec![ForgeAdminCLI, SiteAgent, Flow, Machineatron], + ); x.perm("DeleteExpectedMachine", vec![ForgeAdminCLI, SiteAgent]); x.perm("UpdateExpectedMachine", vec![ForgeAdminCLI, SiteAgent]); x.perm("CreateExpectedMachines", vec![ForgeAdminCLI, SiteAgent]); diff --git a/crates/machine-a-tron/Dockerfile b/crates/machine-a-tron/Dockerfile new file mode 100644 index 0000000000..bfb77dd48d --- /dev/null +++ b/crates/machine-a-tron/Dockerfile @@ -0,0 +1,89 @@ +# syntax=docker/dockerfile:1.7 +# +# Cross-compilation: the BUILDER always runs NATIVE on the build host +# ($BUILDPLATFORM) and cross-compiles to x86_64-unknown-linux-gnu. This is not +# a Mac convenience: building the amd64 stage the naive way on an Apple +# Silicon host runs rustc under QEMU emulation, and QEMU SEGFAULTs assembling +# aws-lc-sys's hand-written x86-64 assembly (s2n-bignum) — even though the +# resulting binary runs fine on real x86_64. On an x86_64 build host the +# builder is simply native and the "cross" compile is a no-op. +# +# Build from the repo root: +# +# docker buildx build \ +# --platform linux/amd64 \ +# -f crates/machine-a-tron/Dockerfile \ +# --build-arg VERSION=0.1.0 \ +# -t machine-a-tron:latest \ +# . +# +ARG RUST_VERSION=1.96.0 + +FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-bookworm AS builder + +ARG VERSION +ARG CI_COMMIT_SHORT_SHA +ENV VERSION=${VERSION} +ENV CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + pkg-config \ + protobuf-compiler \ + libprotobuf-dev \ + && if [ "$(uname -m)" != "x86_64" ]; then \ + apt-get install -y --no-install-recommends \ + gcc-x86-64-linux-gnu libc6-dev-amd64-cross; \ + else \ + apt-get install -y --no-install-recommends gcc libc6-dev; \ + fi \ + && rm -rf /var/lib/apt/lists/* + +RUN rustup target add x86_64-unknown-linux-gnu + +ENV CARGO_HOME=/cargo-home +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +ENV CARGO_TARGET_DIR=/cargo-target +ENV RUST_BACKTRACE=1 + +WORKDIR /workspace +COPY . . + +# Build release binary with cross-compilation +RUN --mount=type=cache,id=nico-mat-cross-cargo-home,target=/cargo-home,sharing=locked \ + --mount=type=cache,id=nico-mat-cross-cargo-target,target=/cargo-target,sharing=locked \ + if [ "$(uname -m)" != "x86_64" ]; then \ + export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \ + CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc; \ + fi && \ + cargo build -p carbide-machine-a-tron --bin machine-a-tron --locked --release \ + --target x86_64-unknown-linux-gnu && \ + mkdir -p /artifacts && \ + cp /cargo-target/x86_64-unknown-linux-gnu/release/machine-a-tron /artifacts/machine-a-tron + +# Runtime image +FROM --platform=linux/amd64 debian:bookworm-slim + +ARG VERSION +ARG CI_COMMIT_SHORT_SHA +LABEL org.opencontainers.image.version="${VERSION}" +LABEL org.opencontainers.image.revision="${CI_COMMIT_SHORT_SHA}" +LABEL org.opencontainers.image.title="machine-a-tron" +LABEL org.opencontainers.image.description="NICo Hardware Simulation Tool" +LABEL org.opencontainers.image.vendor="NVIDIA" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + iproute2 \ + iputils-ping \ + libssl3 \ + libudev1 \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /opt/machine-a-tron/bin /opt/machine-a-tron/templates /tmp/machine-a-tron-data + +COPY --from=builder /artifacts/machine-a-tron /opt/machine-a-tron/bin/machine-a-tron +COPY crates/machine-a-tron/templates /opt/machine-a-tron/templates + +WORKDIR /opt/machine-a-tron +ENTRYPOINT ["/opt/machine-a-tron/bin/machine-a-tron"] diff --git a/dev/deployment/devspace/README.md b/dev/deployment/devspace/README.md index 57793b8c7a..4e460e8a6f 100644 --- a/dev/deployment/devspace/README.md +++ b/dev/deployment/devspace/README.md @@ -86,9 +86,8 @@ devspace deploy DevSpace will: - build the local runtime images from [`Dockerfile.api`](Dockerfile.api), [`Dockerfile.bmc-proxy`](Dockerfile.bmc-proxy), and [`Dockerfile.machine-a-tron`](Dockerfile.machine-a-tron) -- deploy the Helm chart in [`helm/`](../../../helm) -- apply the local-only `machine-a-tron` Kubernetes objects from [`machine-a-tron.yaml`](machine-a-tron.yaml) with `kubectl` -- inject the built image names and DevSpace-generated tags into both deployments at runtime +- deploy the Helm chart in [`helm/`](../../../helm) (including `nico-machine-a-tron`) +- inject the built image names and DevSpace-generated tags into all deployments at runtime The image builds are configured in [`devspace.yaml`](../../../devspace.yaml). The Dockerfiles are multi-stage builds: the builder stage compiles the Rust binary inside Docker from the local `build-container-localdev` image, and the runtime stage copies only the finished binary and required runtime assets. DevSpace first checks whether `build-container-localdev` already exists locally and reuses it if present; otherwise it builds it from [`dev/docker/Dockerfile.build-container-x86_64`](../../../dev/docker/Dockerfile.build-container-x86_64). BuildKit cache mounts are used for Cargo registry, Cargo git checkouts, and Cargo target output so rebuilds stay fast without copying host build artifacts into the image. @@ -96,7 +95,7 @@ The DevSpace images also use Dockerfile-specific ignore files: [`Dockerfile.api. DevSpace watches the Rust workspace, toolchain metadata, and the runtime Dockerfiles to decide when images need rebuilding. -The production Helm chart is still only responsible for the product services. `machine-a-tron` is deployed separately as plain local-only Kubernetes objects in [`machine-a-tron.yaml`](machine-a-tron.yaml), with DevSpace wiring in the local image tag and certificate issuer from [`devspace.yaml`](../../../devspace.yaml). The local API and BMC proxy configs in [`values.base.yaml`](values.base.yaml) point BMC traffic at `machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266`. +The `nico-machine-a-tron` Helm subchart configuration is in [`values.base.yaml`](values.base.yaml). The local API and BMC proxy configs also point BMC traffic at `nico-machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266`. Common usage: @@ -117,7 +116,11 @@ docker build -t "nico-bmc-proxy:" -f dev/deployment/devs docker build -t "machine-a-tron:" -f dev/deployment/devspace/Dockerfile.machine-a-tron . ``` -DevSpace then deploys the Helm chart with the built `nico-api` image wired into `global.image.repository` and `global.image.tag`, the built `nico-bmc-proxy` image wired into the `nico-bmc-proxy` chart values, and applies the local-only `machine-a-tron` manifest with its image wired into the `Deployment` spec. +DevSpace then deploys the Helm chart with: +- the built `nico-api` image wired into `global.image.repository` and `global.image.tag` +- the built `nico-bmc-proxy` image wired into the `nico-bmc-proxy` chart values +- the built `machine-a-tron` image wired into the `nico-machine-a-tron` chart values +- certificate issuer settings from the DevSpace environment variables ## Re-initializing infra-controller to a clean slate @@ -129,7 +132,7 @@ You can start over again (purging the resources from k8s) by running: devspace purge -n nico-system ``` -and it will delete the NICo Helm release and machine-a-tron deployments. +and it will delete the NICo Helm release (including machine-a-tron). To clear out the nico database to start from scratch again, run the nuke-postgres.sh helper script: diff --git a/dev/deployment/devspace/machine-a-tron.yaml b/dev/deployment/devspace/machine-a-tron.yaml deleted file mode 100644 index 0edf1cfe6f..0000000000 --- a/dev/deployment/devspace/machine-a-tron.yaml +++ /dev/null @@ -1,140 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: machine-a-tron ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: machine-a-tron-config -data: - mat.toml: | - carbide_api_url = "https://nico-api.nico-system.svc.cluster.local:1079" - interface = "NOTUSED" - tui_enabled = false - use_pxe_api = true - bmc_mock_host_tar = "/opt/machine-a-tron/dell_poweredge_r750.tar.gz" - bmc_mock_dpu_tar = "/opt/machine-a-tron/nvidia_dpu.tar.gz" - bmc_mock_port = 1266 - use_single_bmc_mock = true - mock_bmc_ssh_server = true - mock_bmc_ssh_port = 2222 - persist_dir = "/tmp/machine-a-tron-data" - register_expected_machines = true - - # machine-a-tron config generated from "just setup-k3s-env-ips" - [machines.config] - host_count = 10 - dpu_per_host_count = 2 - dpu_reboot_delay = 1 # in units of seconds - host_reboot_delay = 1 # in units of seconds - vpc_count = 0 - admin_dhcp_relay_address = "192.168.176.1" - oob_dhcp_relay_address = "192.168.192.1" - subnets_per_vpc = 0 - run_interval_working = "1s" - run_interval_idle = "10s" - template_dir = "/opt/machine-a-tron/templates" ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: machine-a-tron-certificate -spec: - duration: 720h0m0s - renewBefore: 360h0m0s - secretName: machine-a-tron-certificate - privateKey: - algorithm: ECDSA - size: 384 - issuerRef: - kind: Issuer - name: local-ca-issuer - group: cert-manager.io - dnsNames: - - machine-a-tron.nico-system.svc.cluster.local - - machine-a-tron-bmc-mock.nico-system.svc.cluster.local - uris: - - spiffe://nico.local/nico-system/sa/machine-a-tron ---- -apiVersion: v1 -kind: Service -metadata: - name: machine-a-tron-bmc-mock - labels: - app: machine-a-tron -spec: - selector: - app: machine-a-tron - ports: - - name: redfish - port: 1266 - targetPort: redfish - protocol: TCP - - name: ssh - port: 22 - targetPort: ssh - protocol: TCP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: machine-a-tron - labels: - app: machine-a-tron -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app: machine-a-tron - template: - metadata: - labels: - app: machine-a-tron - spec: - serviceAccountName: machine-a-tron - containers: - - name: machine-a-tron - image: machine-a-tron - imagePullPolicy: IfNotPresent - command: - - /opt/machine-a-tron/bin/machine-a-tron - env: - - name: FORGE_ROOT_CA_PATH - value: /var/run/secrets/spiffe.io/ca.crt - - name: CLIENT_CERT_PATH - value: /var/run/secrets/spiffe.io/tls.crt - - name: CLIENT_KEY_PATH - value: /var/run/secrets/spiffe.io/tls.key - - name: MACHINE_A_TRON_CONFIG_PATH - value: /opt/machine-a-tron/config/mat.toml - ports: - - name: redfish - containerPort: 1266 - - name: ssh - containerPort: 2222 - readinessProbe: - tcpSocket: - port: redfish - initialDelaySeconds: 5 - periodSeconds: 10 - volumeMounts: - - name: spiffe - mountPath: /var/run/secrets/spiffe.io - readOnly: true - - name: config - mountPath: /opt/machine-a-tron/config - readOnly: true - - name: persist - mountPath: /tmp/machine-a-tron-data - volumes: - - name: spiffe - secret: - secretName: machine-a-tron-certificate - - name: config - configMap: - name: machine-a-tron-config - - name: persist - emptyDir: {} diff --git a/dev/deployment/devspace/values.base.yaml b/dev/deployment/devspace/values.base.yaml index cba862e532..efeef49369 100644 --- a/dev/deployment/devspace/values.base.yaml +++ b/dev/deployment/devspace/values.base.yaml @@ -139,6 +139,21 @@ nico-bmc-proxy: client_cert = "/var/run/secrets/spiffe.io/tls.crt" client_key = "/var/run/secrets/spiffe.io/tls.key" +nico-machine-a-tron: + enabled: true + namespaceOverride: "nico-mat" + machineATron: + nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" + pods: + default: + machines: + config: + hwType: wiwynn_gb200_nvl + hostCount: 10 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "192.168.192.1" + adminDhcpRelayAddress: "192.168.176.1" + nico-dhcp: enabled: false nico-dns: diff --git a/devspace.yaml b/devspace.yaml index ef66eff130..33adc87e80 100644 --- a/devspace.yaml +++ b/devspace.yaml @@ -79,42 +79,24 @@ deployments: repository: ${runtime.images.nico-api.image} tag: ${runtime.images.nico-api.tag} pullPolicy: IfNotPresent + certificate: + issuerRef: + kind: ${LOCAL_DEV_CERT_ISSUER_KIND} + name: ${LOCAL_DEV_CERT_ISSUER_NAME} + group: ${LOCAL_DEV_CERT_ISSUER_GROUP} nico-bmc-proxy: image: repository: ${runtime.images.nico-bmc-proxy.image} tag: ${runtime.images.nico-bmc-proxy.tag} pullPolicy: IfNotPresent + nico-machine-a-tron: + image: + repository: ${runtime.images.machine-a-tron.image} + tag: ${runtime.images.machine-a-tron.tag} + pullPolicy: IfNotPresent valuesFiles: - dev/deployment/devspace/values.base.yaml - dev/deployment/devspace/values.generated.yaml - machine-a-tron-local: - namespace: ${LOCAL_DEV_NAMESPACE} - updateImageTags: true - kubectl: - manifests: - - dev/deployment/devspace/machine-a-tron.yaml - patches: - - target: - apiVersion: cert-manager.io/v1 - kind: Certificate - name: machine-a-tron-certificate - op: replace - path: spec.issuerRef.kind - value: ${LOCAL_DEV_CERT_ISSUER_KIND} - - target: - apiVersion: cert-manager.io/v1 - kind: Certificate - name: machine-a-tron-certificate - op: replace - path: spec.issuerRef.name - value: ${LOCAL_DEV_CERT_ISSUER_NAME} - - target: - apiVersion: cert-manager.io/v1 - kind: Certificate - name: machine-a-tron-certificate - op: replace - path: spec.issuerRef.group - value: ${LOCAL_DEV_CERT_ISSUER_GROUP} hooks: - name: load-images-into-local-cluster @@ -129,5 +111,11 @@ hooks: "${runtime.images.nico-bmc-proxy.image}:${runtime.images.nico-bmc-proxy.tag}" \ "${runtime.images.machine-a-tron.image}:${runtime.images.machine-a-tron.tag}" ;; + k3s-*) + echo "Loading images into k3s cluster..." + docker save "${runtime.images.nico-api.image}:${runtime.images.nico-api.tag}" | sudo k3s ctr images import - + docker save "${runtime.images.nico-bmc-proxy.image}:${runtime.images.nico-bmc-proxy.tag}" | sudo k3s ctr images import - + docker save "${runtime.images.machine-a-tron.image}:${runtime.images.machine-a-tron.tag}" | sudo k3s ctr images import - + ;; # Add new local K8s tools here esac diff --git a/docs/development/machine-a-tron-deployment.md b/docs/development/machine-a-tron-deployment.md new file mode 100644 index 0000000000..077acd6a96 --- /dev/null +++ b/docs/development/machine-a-tron-deployment.md @@ -0,0 +1,367 @@ +# machine-a-tron — Build & Deployment Guide + +machine-a-tron is a bare-metal simulator for NICo testing. It hosts mock DPUs and +servers via Redfish BMC, allowing end-to-end NICo flows without real hardware. This +guide documents everything needed to build the container image and deploy it on a +cluster. + +## Overview + +machine-a-tron runs in **Override Mode**: site-explorer redirects all Redfish traffic +to the mock BMC server running inside the pod. It is only suitable for +simulation-only clusters (no real hardware). + +## Quick path: `setup-machine-a-tron.sh` + +For a running NICo Core site, the fastest and most reliable way to deploy is the +end-to-end script, which performs every step in this guide (namespace, pull +secret, CA/Vault secret refresh, BMC credential seeding, `bmc_proxy` +configuration, DHCP-pool sizing, cert reissue, deploy, and verification) and is +idempotent: + +```bash +export KUBECONFIG=/path/to/kubeconfig +export REGISTRY_PULL_SECRET= # only if the pull secret is absent +helm-prereqs/setup-machine-a-tron.sh # add -y for non-interactive +``` + +## Quick start: a 4500-host simulated fleet + +Build and push the image to the registry of your choice, then run the setup +script in scale mode. Nothing registry-specific is committed — the image +location, tag, and pull credentials all come from the environment, exactly +like `setup.sh`: + +```bash +export KUBECONFIG=/path/to/site/kubeconfig +export NICO_IMAGE_REGISTRY=/ # e.g. registry.example.com/nico +export MAT_IMAGE_TAG= # tag you built and pushed (see §2) +export REGISTRY_PULL_SECRET= # omit if the pull secret already exists + +MAT_MODE=scale HOST_COUNT=4500 helm-prereqs/setup-machine-a-tron.sh -y +``` + +That is the whole procedure. The script exits after its bounded verification +window while ingestion continues in-cluster; progress is visible with: + +```bash +kubectl exec -n postgres -- su postgres -c \ + "psql -d nico_system_nico -tAc \"SELECT + (SELECT count(*) FROM explored_endpoints) || ' explored / ' || + (SELECT count(*) FROM machines) || ' machines';\"" +``` + +### What to expect (measured on a 3-node dev cluster, dev-sized postgres) + +4500 hosts × 2 DPUs = 13,500 BMC endpoints → 13,500 machines. + +| Phase | Duration | Notes | +|---|---|---| +| Deploy + DHCP registration | ~60–90 min | ~150–180 interfaces/min while 13.5k mock FSMs boot | +| Exploration sweep | ~3–5 h | overlaps DHCP; `explorations_per_run=120` per cycle | +| Preingestion | tracks the sweep | ~90% conversion, completes shortly after it | +| Identification + creation | final ~2–3 h | hosts identify in waves; creation drains at ~150–300 machines per explore cycle | +| **End to end** | **~6–9 h, unattended** | 100 hosts ≈ 12 min and 1000 hosts ≈ 25 min, for calibration | + +The pipeline is autonomous once the script completes — it has run through +multi-hour client connectivity outages without intervention. Occasional +nico-api restarts under peak ingestion load are absorbed by the pipeline +(machines resume within a cycle). Re-running the script is always safe +(idempotent) and re-registers any expected machines that arrived late. + +The rest of this document explains what that script does and why, and is the +reference for manual deployment or debugging. The script's header comments +enumerate the non-obvious failure modes it guards against. + +## 1. Prerequisites + +- Docker with `buildx` and a `linux/arm64` builder available (see below) +- A container registry you can push to and the cluster can pull from + (referenced via `NICO_IMAGE_REGISTRY`, same convention as `setup.sh`) +- A cluster where machine-a-tron can reach `nico-api.nico-system.svc.cluster.local:1079` +- The `nico-machine-a-tron` Helm chart (`helm/charts/nico-machine-a-tron`) +- The image pull secret `machine-a-tron-pull` in the `nico-mat` namespace + (created automatically by the setup script from `REGISTRY_PULL_SECRET`) + +## 2. Building the Container Image + +### Why cross-compilation is required + +machine-a-tron must run on **x86_64** cluster nodes. The Rust dependency `aws-lc-sys` +contains hand-written x86_64 assembly (`s2n-bignum`). Compiling under QEMU emulation +causes a SIGSEGV in the assembler (`bignum_madd_n25519.S`). We therefore use true +cross-compilation: a native `linux/arm64` Rust compiler targeting +`x86_64-unknown-linux-gnu`. + +The `carbide-rpc` crate runs a protobuf build script that requires both `protoc` +and the protobuf well-known types (`libprotobuf-dev` on Debian). `libredfish` is a +Git dependency so `git` must also be present in the build stage. + +### Build command + +Run from the **repository root**: + +```bash +# Same convention as setup.sh: your registry/repository prefix, no scheme. +REGISTRY=${NICO_IMAGE_REGISTRY:?export NICO_IMAGE_REGISTRY=/} +COMMIT=$(git rev-parse --short HEAD) +TAG="${COMMIT}-amd64" + +docker buildx build \ + --platform linux/amd64 \ + -f crates/machine-a-tron/Dockerfile \ + --push \ + -t "${REGISTRY}/machine-a-tron:${TAG}" \ + . +``` + +> **Note**: `--load` does not work for cross-platform builds — use `--push` directly. +> The build takes ~4–5 minutes on a cold cache; subsequent builds with warm cache are +> under 30 seconds. + +### Registry authentication + +```bash +docker login "${NICO_IMAGE_REGISTRY%%/*}" \ + -u "${REGISTRY_PULL_USERNAME:-\$oauthtoken}" \ + -p "${REGISTRY_PULL_SECRET}" +``` + +Some registries use a fixed username with API-key auth — set `REGISTRY_PULL_USERNAME` accordingly (default: `$oauthtoken`). + +## 3. Cluster Prerequisites + +### Namespaces and secrets + +```bash +# Create the deployment namespace +kubectl create namespace nico-mat --dry-run=client -o yaml | kubectl apply -f - + +# Image pull secret (replace with your NVIDIA API key) +kubectl create secret docker-registry machine-a-tron-pull \ + -n nico-mat \ + --docker-server="${NICO_IMAGE_REGISTRY%%/*}" \ + --docker-username="${REGISTRY_PULL_USERNAME:-\$oauthtoken}" \ + --docker-password="${REGISTRY_PULL_SECRET}" \ + --dry-run=client -o yaml | kubectl apply -f - + +# Copy nico-roots CA secret from nico-system +kubectl get secret nico-roots -n nico-system -o json \ + | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid,.metadata.creationTimestamp)' \ + | kubectl apply -n nico-mat -f - +``` + +### Vault secrets + +machine-a-tron's service account needs the Vault secrets present in its namespace +(even though they are optional — their absence produces confusing log noise): + +```bash +for secret in nico-vault-approle-tokens nico-vault-token vault-cluster-info; do + kubectl get secret "$secret" -n nico-system -o json \ + | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid,.metadata.creationTimestamp)' \ + | kubectl apply -n nico-mat -f - +done +``` + +> **Critical after a site reprovision:** re-copy `nico-roots` from `nico-system` +> as well. A reprovision recreates the nico-system CA; a machine-a-tron carried +> over from before will trust the *old* CA (stale `nico-roots`) and present a +> client cert signed by the old CA, so **every** mTLS call to nico-api fails +> with `client error (Connect)`. Re-copy the CA and delete the old cert secret +> (`nico-machine-a-tron-certificate`) so cert-manager reissues from the current +> CA: +> ```bash +> kubectl get secret nico-roots -n nico-system -o json \ +> | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid,.metadata.creationTimestamp,.metadata.ownerReferences)' \ +> | kubectl apply -n nico-mat -f - +> kubectl delete secret nico-machine-a-tron-certificate -n nico-mat --ignore-not-found +> ``` + +### BMC credentials in Vault (required for site-explorer) + +site-explorer's `check_preconditions` requires three site-default Vault +credentials before it will explore any endpoint: +`machines/bmc/site/root`, `machines/all_dpus/site_default/uefi-metadata-items/auth`, +and `machines/all_hosts/site_default/uefi-metadata-items/auth`. The two UEFI +paths are created by the nico-prereqs `kvSeeds` but with **empty passwords**, +which fails the check (`vault does not have a valid password entry`) — they +must be re-seeded with any non-empty password. `machines/bmc/site/root` is not +seeded at all; without it every run aborts with `MissingCredentials`. + +Beyond the preconditions, the **credential rotation flow** requires this exact +chain (all handled by `setup-machine-a-tron.sh` Phase 4): + +| Vault path | Value | Why | +|---|---|---| +| `machines/all_hosts/factory_default/bmc-metadata-items/dell` | `root`/`factory_password` | Host BMC factory default (mock's `DUMMY_FACTORY_PASSWORD`). Path segment is **lowercase** `dell` — `BMCVendor`'s `Display` impl lowercases. | +| `machines/all_dpus/factory_default/bmc-metadata-items/root` | `root`/`0penBmc` | DPU BMC factory default (mock's `DUMMY_FACTORY_DPU_PASSWORD`) — note it differs from the host factory password. | +| `machines/bmc/site/root` | `root`/<distinct> | Rotation target. **Must differ from both factory passwords**, or the rotation is a no-op and the mock rejects with `403 Factory-default password must be changed` forever. | + +site-explorer logs into each BMC with its factory default, rotates the password +to the site root value, then proceeds — using the wrong factory password (or a +site root equal to factory) yields `401 Unauthorized`, which latches a +self-perpetuating `AvoidLockout` (NICO-SITEEXPLORER-144) until an operator +clears it (`nico-admin-cli site-explorer refresh `). + +## 4. Deploying the Helm Chart + +### Site values file + +Copy `helm-prereqs/values/machine-a-tron.yaml` and fill in the site-specific values: + +| Field | Description | +|---|---| +| `image.tag` | Tag produced by step 2 (e.g. `8c35783af-amd64`) | +| `machines.dell-hosts.oobDhcpRelayAddress` | Gateway of the OOB/underlay network from nico-core site config | +| `machines.dell-hosts.adminDhcpRelayAddress` | Gateway of the admin network from nico-core site config | +| `machines.dell-hosts.hostCount` | Must not exceed available OOB DHCP addresses (`hostCount + hostCount×dpuPerHostCount`) | + +### Critical: SPIFFE URI override + +The cert-manager `Certificate` resource auto-generates a SPIFFE URI based on the +deployment namespace: `spiffe://nico.local/nico-mat/sa/nico-machine-a-tron`. + +nico-api's `spiffe_service_base_paths` only includes `/nico-system/sa/` (and two +others), so this URI is **not recognized**. The result is that machine-a-tron's +principal is only `TrustedCertificate` — not `SpiffeServiceIdentifier("machine-a-tron")` +— and every gRPC call beyond `Version` returns HTTP 403. + +The values file already includes the fix: + +```yaml +certificate: + uris: + - "spiffe://nico.local/nico-system/sa/machine-a-tron" +``` + +This overrides the auto-generated URI so nico-api can correctly identify and authorize +machine-a-tron as the `Machineatron` RBAC principal. + +### Deploy + +```bash +helm upgrade --install nico-machine-a-tron \ + helm/charts/nico-machine-a-tron \ + -n nico-mat \ + --create-namespace \ + -f helm-prereqs/values/machine-a-tron.yaml +``` + +## 5. Configuring Override Mode + +Configure nico-core's site-explorer to redirect all Redfish traffic to the mock. +Add this to the nico-core site config under `[site_explorer]`: + +```toml +[site_explorer] +bmc_proxy = "nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local:1266" +``` + +> **Use the cross-namespace FQDN.** site-explorer runs inside nico-api in +> `nico-system`; a bare service name resolves against that namespace and fails +> ("connection refused" on every Redfish call) because the mock's Service lives +> in `nico-mat`. +> +> **This setting does not survive a nico-core `helm upgrade`** — the ConfigMap +> is chart-owned, so an upgrade silently reverts it. Re-run +> `setup-machine-a-tron.sh` (idempotent) after any nico-core upgrade. + +> **Field name matters.** The config field is `bmc_proxy` — a single +> `"host:port"` string (`crates/site-explorer/src/config.rs`). The older +> `override_target_ip` / `override_target_port` fields are **deprecated**, and +> `override_target_host` was never a valid field at all (earlier revisions of +> this guide were wrong — a value under that key is silently ignored). + +Setting `bmc_proxy` at launch also makes `allow_changing_bmc_proxy` default to +`true`. That is what allows the chart value `machineATron.configureBmcProxyHost` +to work: when set, machine-a-tron calls nico-api's `set_dynamic_config` to set +`bmc_proxy` at runtime, but that call is rejected with `PermissionDenied` unless +`allow_changing_bmc_proxy` is true. The two mechanisms are complementary — the +values file ships `configureBmcProxyHost: +"nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local"` (FQDN, same +cross-namespace requirement), and the nico-core `bmc_proxy` setting both +enables that path and covers the case where the runtime call has not happened +yet. + +Apply via `helm upgrade` of the nico-core chart, or patch the configmap and +restart nico-api (site-explorer runs in-process in nico-api): + +```bash +kubectl rollout restart deployment/nico-api -n nico-system +``` + +### Why machines get created (expected_machines) + +site-explorer's `MachineCreator` refuses to create a managed host unless a +matching `expected_machines` row exists (by BMC MAC) — otherwise it logs +`Refusing to create managed host, expected machines entry not found`. machine-a-tron +auto-registers these when `machineATron.registerExpectedMachines: true` (the +default in the values file). DHCP discovery alone is **not** sufficient. + +## 6. Verifying Startup + +Check that machine-a-tron passes the initial API calls: + +```bash +kubectl logs -n nico-mat deployment/nico-machine-a-tron | grep -E "firmware|Got desired|Error:" +``` + +Expected: `Got desired firmware versions from the server: [...]` + +Check nico-api for denied requests (should be empty after the SPIFFE fix): + +```bash +kubectl logs -n nico-system deployment/nico-api | grep "Request denied.*machine-a-tron" +``` + +## 7. DHCP Address Space + +machine-a-tron allocates one OOB IP per BMC interface (1 per host + 1 per DPU). With +5 hosts and 2 DPUs each that is **15 IPs** (`5 + 5×2`). Ensure the OOB DHCP prefix in +nico-core is large enough. Note the usable count is smaller than the raw CIDR: a +`/28` (16 addresses) yields only ~10–13 usable after subtracting network, +broadcast, gateway, and any reserved addresses — on dev6 a `/28` yielded 10, so +`hostCount: 3 × 2 DPUs = 9` fit but `5 × 2 = 15` did not. Use at least a `/27` +for the default counts, or reduce `hostCount` / `dpuPerHostCount`. Symptom of +overflow: `No IP addresses left in prefix ...` and machines stuck in `BmcInit`. + +If the prefix is exhausted from previous runs, either: + +- Force-delete old machine records via the admin CLI, or +- Reprovision the site (which clears the database) and redeploy from scratch. + +> **Do NOT hand-delete rows** from `machine_interfaces`, `dhcp_entries`, or +> `machine_interface_addresses` to free leases. The `machine_dhcp_records` view +> inner-joins the singleton control row `machine_interfaces_deletion` (id=1); if +> that row is deleted (easy to do by accident when clearing related tables) the +> view returns zero rows and `DiscoverDhcp` fails for **every** BMC with +> `Database Error: no rows returned by a query that expected to return at least +> one row`. If you hit that, restore the row: +> ```sql +> INSERT INTO machine_interfaces_deletion (id) VALUES (1) ON CONFLICT (id) DO NOTHING; +> ``` + +## 8. Summary of Non-Obvious Fixes Discovered + +| Problem | Root cause | Fix | +|---|---|---| +| `exec format error` in pod | Image was built for `arm64`, nodes are `x86_64` | Cross-compile with `--platform linux/amd64` and `x86_64-unknown-linux-gnu` Rust target | +| `SIGSEGV` compiling `aws-lc-sys` | QEMU emulates the `.S` assembler, which crashes | True cross-compilation (native arm64 host → x86_64 target) instead of QEMU | +| `File not found: google/protobuf/timestamp.proto` | `libprotobuf-dev` absent in build image | Add `libprotobuf-dev` to `apt-get install` in builder stage | +| `git fetch ... (exit status: 127)` | `libredfish` is a git dependency, `git` not in slim image | Add `git` to builder stage | +| `--load` fails for cross-platform builds | Docker limitation | Use `--push` directly to registry | +| HTTP 403 on every gRPC call | machine-a-tron cert SPIFFE URI not in nico-api's `service_base_paths` | Set `certificate.uris: ["spiffe://nico.local/nico-system/sa/machine-a-tron"]` in values | +| `client error (Connect)` on every nico-api call after a reprovision | Stale `nico-roots` CA + client cert signed by the old CA | Re-copy `nico-roots` from nico-system; delete `nico-machine-a-tron-certificate` so cert-manager reissues from the current CA | +| site-explorer aborts with `MissingCredentials machines/bmc/site/root` | Site BMC root cred not in default `kvSeeds` | Seed `secrets/machines/bmc/site/root` = `root`/<non-factory password> in Vault | +| site-explorer aborts with `MissingCredentials .../uefi-metadata-items/auth` | kvSeeds create the UEFI creds with **empty** passwords, which fail validation | Re-seed both site_default UEFI creds with any non-empty password | +| Redfish redirect ignored; `endpoint_explorations=0` | Wrong config field (`override_target_host` is not real) | Use `bmc_proxy = "nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local:1266"` under `[site_explorer]` | +| Redfish `connection refused` on every endpoint despite bmc_proxy set | Bare service name resolves against nico-system, not nico-mat | Use the cross-namespace FQDN in `bmc_proxy` | +| Host BMCs 401 while DPUs explore fine | Host and DPU factory passwords differ (`factory_password` vs `0penBmc`); host factory cred missing or wrong | Seed `machines/all_hosts/factory_default/bmc-metadata-items/dell` = `root`/`factory_password` (lowercase `dell`) | +| DPU explorations stuck at `403 Factory-default password must be changed` | Site root password equals the factory password → rotation is a no-op | Seed `machines/bmc/site/root` with a password distinct from both factory defaults | +| All endpoints latch `AvoidLockout` (NICO-SITEEXPLORER-144) after a cred fix | A previous Unauthorized is self-perpetuating in the exploration report | `nico-admin-cli site-explorer refresh ` per endpoint (or re-run setup-machine-a-tron.sh, which clears it) | +| `Refusing to create managed host`; machine-a-tron logs `PermissionDenied` on registration | nico-api build lacks the `Machineatron` → `AddExpectedMachine` RBAC grant | Rebuild nico-api with the grant (internal_rbac_rules.rs); the setup script also has a DB fallback | +| Machine creation fails `No IP addresses left in prefix ` | Admin pool too small: creation needs one host-PF IP per DPU | Fit `hostCount×dpuPerHostCount` ≤ usable admin-pool IPs (the script auto-fits) | +| `Refusing to create managed host, expected machines entry not found` | No `expected_machines` row for the discovered BMC MAC | Set `machineATron.registerExpectedMachines: true` (default) so machine-a-tron auto-registers them | +| `No IP addresses left in prefix ...`; machines stuck in `BmcInit` | OOB DHCP pool too small for host×DPU count | Sizing: `hostCount + hostCount×dpuPerHostCount` ≤ usable pool IPs; use ≥ /27 or reduce counts | +| `DiscoverDhcp`: `no rows ... expected to return at least one row` | `machine_interfaces_deletion` singleton (id=1) deleted; breaks `machine_dhcp_records` view | `INSERT INTO machine_interfaces_deletion (id) VALUES (1) ON CONFLICT DO NOTHING;` — never hand-delete lease rows | diff --git a/docs/development/machine-a-tron-scale-testing.md b/docs/development/machine-a-tron-scale-testing.md new file mode 100644 index 0000000000..0088a9c561 --- /dev/null +++ b/docs/development/machine-a-tron-scale-testing.md @@ -0,0 +1,168 @@ +# Scaling NICo with machine-a-tron: 100 → 1000 → 4500 simulated hosts + +> **Status: DRAFT — early feedback wanted.** Stage 1 (100 hosts × 2 DPUs = +> 300 BMCs → 264 machines) is complete on dev6. Stage 2 (1000 hosts) is +> running as this is written. Stage 3 targets 4500 hosts × 2 DPUs = 13,500 +> BMC endpoints, in support of scaling NICo to ~4500 nodes. +> +> **Rebased on #2764**: this work now sits on top of Alexander's ClusterIP +> migration, which removes nginx/MetalLB from the chart entirely (per-BMC +> ClusterIP Services + ServiceCIDR, multi-pod sharding via `pods..cidr`). +> That migration supersedes issues 10, 12 and 13 below (kept for the record — +> they document why the nginx/MetalLB path was abandoned) and independently +> confirms the direction of the proxy-direct pivot. The scripts' scale mode +> (`bmc_proxy` + client-injected Forwarded) remains valid with `bmcServices` +> disabled and is what all stage results below were measured with. +> +> Branch: `machine-a-tron-e2e-on-2764`. Everything below is reproducible +> with two commands: +> +> ```bash +> export KUBECONFIG=/path/to/site/kubeconfig +> helm-prereqs/cleanup-machine-a-tron.sh -y +> MAT_MODE=scale HOST_COUNT=1000 helm-prereqs/setup-machine-a-tron.sh -y +> ``` + +## What this work delivers + +1. **`helm-prereqs/setup-machine-a-tron.sh`** — one idempotent script that + takes a running NICo site from nothing to created machines: namespace, + pull secret, CA/Vault secret refresh, the full BMC/UEFI credential chain, + nico-core site-config changes, DHCP pool sizing with auto-fit, DB safety + checks, helm deploy, and a verification loop that actively shepherds the + ingestion pipeline (details below on why that is necessary). +2. **`helm-prereqs/cleanup-machine-a-tron.sh`** — the full inverse, so + from-scratch runs are reproducible (this caught several + "works-second-time-only" bugs). +3. **`MAT_MODE=scale`** — a scale profile + (`helm-prereqs/values/machine-a-tron-scale.yaml`) using a **proxy-direct** + transport architecture (see next section), simulated network segments + sized for 13.5k endpoints, and raised site-explorer throughput knobs. +4. A one-line RBAC fix in nico-api (`Machineatron` was missing the + `AddExpectedMachine` grant) plus chart fixes to the nginx/MetalLB mode. + +## The architecture decision: proxy-direct + +The chart offers an nginx/MetalLB mode for large scale: one LoadBalancer +Service per simulated BMC (cap 16,384), nginx terminating TLS and routing to +the mock. We started there and hit four independent failure modes at just 300 +endpoints (§ issues 10–13). The pivotal realization: + +**`site_explorer.bmc_proxy` alone already scales.** When it is set, the +Redfish client itself injects `Forwarded: host=` (RFC 7239, +`crates/redfish/src/libredfish/implementation.rs`), and the mock's shared +registry (`use_single_bmc_mock = true`) routes each request to the right +simulated BMC. One ClusterIP Service carries the whole fleet — no nginx, no +MetalLB pool, no per-BMC Services, no `externalTrafficPolicy: Local` +pitfalls. + +The nginx/MetalLB mode remains the right choice when simulated BMCs must +coexist with **real hardware** (each mock needs a real routable IP). For a +simulation-only cluster it only adds moving parts. The chart fixes we made to +that mode are kept for its real users. + +Result at 300 endpoints: exploration went from constant flapping +(Unreachable/ConnectionRefused under load) to rock-stable 300/300. + +## Complete issue log + +Every issue below was found live on dev6 and is fixed on the branch, encoded +in the scripts/charts with explanatory comments. + +### Baseline (override-mode) end-to-end + +| # | Issue | Root cause | Fix | +|---|---|---|---| +| 1 | Every nico-api call fails `client error (Connect)` after a site reprovision | machine-a-tron trusts the old CA (stale `nico-roots` copy) and presents a cert signed by it | Script refreshes `nico-roots` + Vault secrets from nico-system and deletes the client-cert secret so cert-manager reissues from the current CA | +| 2 | Redfish redirect silently ignored | Docs said `override_target_host` — never a valid field; the real field is `bmc_proxy = "host:port"`, and it must be the **cross-namespace FQDN** (site-explorer runs in nico-system; a bare service name doesn't resolve) | Script sets `bmc_proxy` correctly; docs fixed | +| 3 | site-explorer aborts every run: `MissingCredentials` | `machines/bmc/site/root` isn't in default kvSeeds; the seeded UEFI creds ship with **empty** passwords which fail validation | Script seeds the full chain | +| 4 | Host BMCs 401 while DPUs explore fine | Host and DPU mock factory passwords differ (`factory_password` vs `0penBmc`); the host factory Vault path vendor segment is **lowercase** (`…/dell` — `BMCVendor`'s `Display` lowercases; the earlier capital-`Dell` seed was read by nobody) | Script seeds both factory creds on the correct paths | +| 5 | machine-a-tron's expected-machine registration 403s (logged misleadingly as "likely already ingested") | `Machineatron` principal missing from the `AddExpectedMachine` RBAC grant — an oversight; it holds the sibling grants (`DiscoverDhcp`, `CreateNetworkSegment`, `GetExpectedSwitch`) | One-line fix in `internal_rbac_rules.rs`; script includes a DB fallback for nico-api builds without it | +| 6 | Endpoints permanently stuck `AvoidLockout` (NICO-SITEEXPLORER-144) on a fresh deploy | Per-MAC rotated creds (`machines/bmc//root`) survive cleanup; a fresh mock is at factory password but the per-MAC entry makes site-explorer present the old rotated one → 401 latch, self-perpetuating by design | Cleanup purges per-MAC creds; setup self-heals stale ones (only when the machine graph is truly empty — machines AND interfaces at 0) | +| 7 | `DiscoverDhcp` fails for every BMC: "no rows returned…" | The `machine_dhcp_records` VIEW inner-joins a singleton control row (`machine_interfaces_deletion` id=1); manual lease cleanup had deleted it | Script restores the singleton; documented: never hand-delete lease rows | +| 8 | Machines never created: admin pool exhausted | Real demand is OOB = `hosts×(1+dpus)` and admin = `hosts×(dpus+1)` (one host-PF IP per DPU **plus one per host at creation**); usable = `2^(32-mask) − reserve_first − 1` | Sizing check with auto-fit; `reserve_first` parsed from the live site config | + +### Scale mode (100 hosts × 2 DPUs and up) + +| # | Issue | Root cause | Fix | +|---|---|---|---| +| 9 | helm deploy aborts: hundreds of `connection reset by peer` | helm's default burst (100 concurrent API calls) overwhelms SOCKS/ssh tunnels when creating hundreds of Services | `--qps 15 --burst-limit 30` (env-overridable) | +| 10 | nginx bmc-proxy CrashLoopBackOff: `host not found in upstream` | Chart template pointed the upstream at the bare chart name, which is not a Service | Point at the `-bmc-mock` Service (chart fix) | +| 11 | Nothing listens on the mock port; probes kill the pod | `use_single_bmc_mock=false` makes each mock bind its **real BMC IP** on the pod netns (bare-metal mode). `true` is the shared-registry mode K8s needs | `useSingleBmcMock: true` | +| 12 | Every registry lookup 404s: `no router configured for host: 10.233.x.x` | nginx forwarded `host=$server_addr`, but kube-proxy DNATs the LB IP to the nginx **pod IP** before the connection arrives | `Forwarded "host=$host"` — the client-requested host is the LB IP end-to-end (chart fix) | +| 13 | LB IPs intermittently Unreachable in-cluster | Per-BMC Services use `externalTrafficPolicy: Local` and the chart's REQUIRED podAffinity stacked all proxies on the mat node | Required anti-affinity between proxy replicas (+ `maxUnavailable=1/maxSurge=0`; with replicas == nodes a surge pod deadlocks the rollout) — chart fix, kept for nginx-mode users | +| 14 | DHCP fails: `No network segment defined for relay addresses` | Config-driven segment creation is **bootstrap-once** — skipped entirely on multi-domain sites ("Multiple domains, skipping initial network creation") | Script clone-inserts the simulated segments from same-type templates; `allocation_strategy` forced to `dynamic` (templates may be `reserved`, which rejects all dynamic DHCP) | +| 15 | AvoidLockout storm on all DPU endpoints; preingestion pinned at exactly `hostCount` | The rotation dance is racy at scale: preingestion's initial BMC reset reboots the mock, which returns at the **factory** password while its per-MAC Vault entry says "rotated" | Pin mock passwords to the site root (`hostBmcPassword`/`dpuBmcPassword`) — site-explorer's documented fallback ("factory failed → sitewide root, no rotation") logs straight in; resets become harmless | +| 16 | Pipeline stalls at preingestion `initial`; manager idle | `waiting_for_explorer_refresh` (set when errors are cleared) gates endpoints out of preingestion and can linger after a healthy report lands (273/300 were parked) | Verification loop unparks endpoints whose reports are clean | +| 17 | Managed hosts identified but machines never created; cycles never finish | `explorations_per_run` was raised to 400 "for throughput" — but identification and creation only run **at the end of a completed explore cycle**, and 400 deep scans per cycle meant cycles stopped completing | Default lowered to 120: cycles complete in ~1–2 min and creation runs every cycle | +| 18 | `Resource pool lo-ip is empty` on the 3rd machine | Machine creation allocates one loopback IP per machine; pool **definitions are seed-once** ("Declaration has drifted since seed … not re-applying") so config widening is ignored; dev6 ships **3** lo-ip addresses | Script inserts free `resource_pool` rows directly for a simulated range (16k) when the pool is smaller than the machine target | + +### A note on the verification loop + +The script's final phase doesn't just poll — it actively shepherds: +re-clears `AvoidLockout`/`Unauthorized` latches (they are one-way by design; +on real hardware an operator runs `nico-admin-cli site-explorer refresh`) and +unparks healthy endpoints. On a simulation cluster with hundreds of +concurrent resets/explorations, transient races are guaranteed; the loop is +the "operator". Mocks have no lockout threshold, so this is safe here. + +## Where we are today + +| Stage | Scale | Result | +|---|---|---| +| Baseline | 1 host × 1 DPU (override mode) | ✅ end-to-end: machines created, full credential rotation exercised | +| Stage 1 | 100 hosts × 2 DPUs = 300 BMCs (proxy-direct) | ✅ 300/300 endpoints stable, machines created and advancing through `hostinit`/`dpuinit` | +| Stage 2 | 1000 hosts × 2 DPUs = 3000 BMCs | ✅ **END TO END OK — 3000/3000 machines** in a single unattended script run (~25 min total; creation ≈ 240 machines/min) | +| Stage 3 | 4500 hosts × 2 DPUs = 13,500 BMCs | ✅ **13,500/13,500 machines — 100% fleet** (first run: 13,500 endpoints explored, 10k+ machines; consolidated rerun on the #2764 ClusterIP chart + a freshly provisioned cluster: every counter at 100% — 13,500 explored / 13,500 preingestion-complete / 4,500 hosts / 13,500 machines, ~15 h wall clock unattended incl. connectivity outages) | + +Stage-3 observations worth reviewers' attention: + +- **The ingestion pipeline is fully autonomous once configured.** Client + connectivity to the cluster dropped twice for extended periods during + stage 3; ingestion continued unattended both times (e.g. +720 machines + through one outage, +4,000 through another). The shepherd loop's latch + clearing — critical in earlier iterations — was a no-op for the entire + stage-3 run thanks to pinned credentials. +- **Measured stage-3 rates on dev6 (3 nodes, dev-sized postgres):** DHCP + ~110 interfaces/min; exploration ~120–360 endpoints/cycle; creation + 40–240 machines per completed explore cycle, sawtoothing with cycle + phasing (identification rebuilds `explored_managed_hosts` each cycle). +- **Per-MAC Vault credential lifecycle needs batching at scale** (issue 19 + below): site-explorer stores one `machines/bmc//root` entry per BMC — + 13,500 entries; deleting them one API round-trip at a time takes hours, + batched server-side it takes seconds. +- `expected_machines` auto-registration worked at stage 3 (9,890+ registered + by machine-a-tron via the API), confirming the RBAC grant path. + +Additional issue found at stage 3: + +| # | Issue | Root cause | Fix | +|---|---|---|---| +| 19 | Stage-2→3 cleanup ran for over an hour "deleting credentials" | One `kubectl exec` per per-MAC Vault deletion × thousands of entries | Batch the deletion loop server-side on the vault pod — one exec total (both cleanup and setup self-heal) | + +## Open questions — feedback wanted + +1. **RBAC**: is granting `Machineatron` → `AddExpectedMachine` acceptable + (commit `9a9ba072a`)? Until a nico-api image with it is deployed, the + script registers expected machines via direct DB insert — okay as a + documented simulation-only fallback? +2. **Seed-once reconcile semantics**: networks, and resource-pool + definitions are all create-once; config changes on established sites are + silently ignored (or warn-only). The script works around this with direct + DB writes (segment clone-insert, pool row insertion). Should NICo support + declarative updates for these instead? +3. **AvoidLockout at scale**: one-way latches are right for real BMCs, but + simulation fleets guarantee latch storms during resets. Worth a + site-config escape hatch (e.g. `site_explorer.lockout_protection = false`) + instead of the script's DB-level clearing? +4. **Mock fidelity**: the mock returns to its configured password after a + BMC reset. Real BMCs persist a rotated password across resets. Should + bmc-mock persist rotated credentials so the rotation path can be exercised + at scale without pinning? +5. **lo-ip per machine**: is one loopback IP per machine the intended + allocation at 13.5k machines, and is there guidance for sizing this pool + in production site templates (dev templates ship 3)? +6. **Cycle economics**: identification/creation only run at the end of a + completed `explore_site` cycle, so `explorations_per_run` trades sweep + throughput against creation latency in a non-obvious way. Worth + documenting (or decoupling creation from the exploration cycle)? diff --git a/helm-prereqs/cleanup-machine-a-tron.sh b/helm-prereqs/cleanup-machine-a-tron.sh new file mode 100755 index 0000000000..e85f7f82e5 --- /dev/null +++ b/helm-prereqs/cleanup-machine-a-tron.sh @@ -0,0 +1,296 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================= +# cleanup-machine-a-tron.sh — tear down machine-a-tron for a from-scratch redeploy +# +# The inverse of setup-machine-a-tron.sh. By default it removes everything that +# script created EXCEPT the namespace and pull secret (so a re-run of setup does +# not need the registry API key again): +# * uninstalls the nico-machine-a-tron Helm release +# * deletes the client cert secret (so cert-manager reissues from the current CA) +# * resets the machine graph in the NICo DB (TRUNCATE ... CASCADE) while +# PRESERVING network config and the machine_interfaces_deletion singleton +# * reverts nico-core site_explorer.bmc_proxy (and any legacy +# override_target_* lines) and restarts nico-api +# * removes the Vault credential machines/bmc/site/root +# +# ONLY for simulation-only clusters. The DB reset truncates machines, +# machine_interfaces, explored_endpoints, explored_managed_hosts, and +# expected_machines with CASCADE — every machine on the cluster is assumed to +# be a machine-a-tron simulation. NEVER run this against a site with real +# hardware inventory. +# +# The machine_interfaces_deletion singleton (id=1) is preserved and its +# last_deletion watermark bumped — deleting it would break the +# machine_dhcp_records view and every subsequent DiscoverDhcp call. +# +# Required environment: +# KUBECONFIG Path to the target cluster kubeconfig. +# Optional environment: +# MAT_NAMESPACE Default: nico-mat +# NICO_SYSTEM_NS Default: nico-system +# POSTGRES_NS Default: postgres +# VAULT_NS Default: vault +# +# Usage: +# export KUBECONFIG=/path/to/kubeconfig +# ./cleanup-machine-a-tron.sh # prompt before each destructive step +# ./cleanup-machine-a-tron.sh -y # non-interactive +# ./cleanup-machine-a-tron.sh --delete-namespace # full teardown (also removes +# # pull secret + copied secrets) +# ./cleanup-machine-a-tron.sh --keep-db # leave the DB untouched +# ./cleanup-machine-a-tron.sh --keep-nico-core-config # leave bmc_proxy in place +# ./cleanup-machine-a-tron.sh --keep-vault-cred # leave BMC cred in Vault +# ============================================================================= + +set -euo pipefail + +MAT_NAMESPACE="${MAT_NAMESPACE:-nico-mat}" +NICO_SYSTEM_NS="${NICO_SYSTEM_NS:-nico-system}" +POSTGRES_NS="${POSTGRES_NS:-postgres}" +VAULT_NS="${VAULT_NS:-vault}" +RELEASE="nico-machine-a-tron" +NICO_DB="nico_system_nico" + +ASSUME_YES=false +DELETE_NAMESPACE=false +KEEP_DB=false +KEEP_NICO_CORE_CONFIG=false +KEEP_VAULT_CRED=false + +for arg in "$@"; do + case "$arg" in + -y|--yes) ASSUME_YES=true ;; + --delete-namespace) DELETE_NAMESPACE=true ;; + --keep-db) KEEP_DB=true ;; + --keep-nico-core-config) KEEP_NICO_CORE_CONFIG=true ;; + --keep-vault-cred) KEEP_VAULT_CRED=true ;; + -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//' | head -70; exit 0 ;; + *) echo "Unknown argument: $arg" >&2; exit 2 ;; + esac +done + +# --- helpers (match setup-machine-a-tron.sh) -------------------------------- +_c() { printf '\033[%sm' "$1"; } +BOLD="$(_c 1)"; RED="$(_c 31)"; GREEN="$(_c 32)"; YEL="$(_c 33)"; BLU="$(_c 34)"; NC="$(_c 0)" +phase() { echo; echo "${BOLD}${BLU}== $* ==${NC}"; } +info() { echo " $*"; } +ok() { echo " ${GREEN}✓${NC} $*"; } +warn() { echo " ${YEL}!${NC} $*" >&2; } +die() { echo "${RED}ERROR:${NC} $*" >&2; exit 1; } +confirm() { $ASSUME_YES && return 0; read -r -p " $* [y/N] " a; [[ "$a" == "y" || "$a" == "Y" ]]; } + +CM_JSON="" +cleanup_tmp() { rm -f "$CM_JSON" 2>/dev/null || true; } +trap cleanup_tmp EXIT + +PG_PRIMARY="" +_pg_primary() { + [[ -n "$PG_PRIMARY" ]] && { echo "$PG_PRIMARY"; return; } + PG_PRIMARY="$(kubectl get pods -n "$POSTGRES_NS" -l application=spilo \ + -o jsonpath='{range .items[*]}{.metadata.name} {.metadata.labels.spilo-role}{"\n"}{end}' 2>/dev/null \ + | awk '$2=="master"{print $1}' | head -1)" + echo "$PG_PRIMARY" +} +psql_run() { # runs SQL, returns raw psql output (not -tAc, so NOTICEs show) + local pg; pg="$(_pg_primary)"; [[ -n "$pg" ]] || die "no Patroni primary in $POSTGRES_NS" + kubectl exec -n "$POSTGRES_NS" "$pg" -- su postgres -c "psql -d $NICO_DB -v ON_ERROR_STOP=1 -c \"$1\"" 2>&1 +} +psql_q() { + local pg; pg="$(_pg_primary)"; [[ -n "$pg" ]] || die "no Patroni primary in $POSTGRES_NS" + kubectl exec -n "$POSTGRES_NS" "$pg" -- su postgres -c "psql -d $NICO_DB -tAc \"$1\"" 2>/dev/null +} +_VAULT_TOKEN="" +vault_cmd() { + if [[ -z "$_VAULT_TOKEN" ]]; then + _VAULT_TOKEN="$(kubectl get secret nico-vault-token -n "$NICO_SYSTEM_NS" -o jsonpath='{.data.token}' | base64 -d)" + [[ -n "$_VAULT_TOKEN" ]] || die "could not read nico-vault-token from $NICO_SYSTEM_NS" + fi + kubectl exec -n "$VAULT_NS" vault-0 -c vault -- sh -c \ + "export VAULT_TOKEN='$_VAULT_TOKEN' VAULT_ADDR=https://127.0.0.1:8200 VAULT_SKIP_VERIFY=true; $1" 2>/dev/null +} + +# ============================================================================= +# Phase 0 — preflight +# ============================================================================= +phase "Phase 0 — preflight" +for t in kubectl helm; do command -v "$t" >/dev/null || die "$t not found in PATH"; done +kubectl cluster-info >/dev/null 2>&1 || die "cannot reach the cluster (check KUBECONFIG)" +ok "cluster reachable" +echo " This will tear down machine-a-tron in namespace ${MAT_NAMESPACE} and reset" +echo " the machine graph in DB ${NICO_DB}. Simulation clusters only." +confirm "Proceed with cleanup?" || die "aborted" + +# ============================================================================= +# Phase 1 — uninstall Helm release +# ============================================================================= +phase "Phase 1 — uninstall Helm release" +if helm status "$RELEASE" -n "$MAT_NAMESPACE" >/dev/null 2>&1; then + # retry — transient "http2: client connection lost" can abort the uninstall + _uninstalled=false + for _i in 1 2 3; do + if helm uninstall "$RELEASE" -n "$MAT_NAMESPACE" >/dev/null 2>&1; then _uninstalled=true; break; fi + helm status "$RELEASE" -n "$MAT_NAMESPACE" >/dev/null 2>&1 || { _uninstalled=true; break; } + warn "uninstall attempt ${_i} failed (transient?) — retrying" + sleep 5 + done + $_uninstalled && ok "release ${RELEASE} uninstalled" || die "could not uninstall ${RELEASE}" +else + ok "release ${RELEASE} not installed" +fi + +# ============================================================================= +# Phase 2 — namespace / cert secret +# ============================================================================= +phase "Phase 2 — namespace / cert secret" +if $DELETE_NAMESPACE; then + if kubectl get ns "$MAT_NAMESPACE" >/dev/null 2>&1; then + confirm "Delete the entire ${MAT_NAMESPACE} namespace (removes pull secret + copied secrets)?" \ + && { kubectl delete namespace "$MAT_NAMESPACE" --wait=false >/dev/null; ok "namespace ${MAT_NAMESPACE} deletion requested"; } \ + || warn "skipped namespace deletion" + else + ok "namespace ${MAT_NAMESPACE} already absent" + fi +else + kubectl delete secret "${RELEASE}-certificate" -n "$MAT_NAMESPACE" --ignore-not-found >/dev/null 2>&1 \ + && ok "client cert secret deleted (cert-manager reissues on next deploy)" + info "namespace + pull secret kept (use --delete-namespace for full teardown)" +fi + +# ============================================================================= +# Phase 3 — reset machine graph in the DB (preserve singleton + network config) +# ============================================================================= +phase "Phase 3 — reset machine graph in DB" +if $KEEP_DB; then + warn "--keep-db set; leaving DB untouched" +else + BEFORE="$(psql_q "SELECT + (SELECT count(*) FROM machines) || '/' || + (SELECT count(*) FROM machine_interfaces) || '/' || + (SELECT count(*) FROM expected_machines) || '/' || + (SELECT count(*) FROM explored_endpoints);")" + info "before (machines/interfaces/expected/endpoints): ${BEFORE:-?}" + if confirm "TRUNCATE machine graph (machines, machine_interfaces, explored_endpoints, explored_managed_hosts, expected_machines) CASCADE?"; then + OUT="$(psql_run "BEGIN; + TRUNCATE machines, machine_interfaces, explored_endpoints, explored_managed_hosts, expected_machines RESTART IDENTITY CASCADE; + UPDATE machine_interfaces_deletion SET last_deletion = now() WHERE id = 1; + INSERT INTO machine_interfaces_deletion (id) VALUES (1) ON CONFLICT (id) DO NOTHING; + COMMIT;")" || die "DB reset failed (rolled back): $OUT" + # surface which tables CASCADE touched + printf '%s\n' "$OUT" | grep -i "truncate cascades" | sed 's/^/ /' || true + AFTER="$(psql_q "SELECT + (SELECT count(*) FROM machines) || '/' || + (SELECT count(*) FROM machine_interfaces) || '/' || + (SELECT count(*) FROM expected_machines) || '/' || + (SELECT count(*) FROM explored_endpoints);")" + SINGLETON="$(psql_q "SELECT count(*) FROM machine_interfaces_deletion WHERE id=1;")" + ok "machine graph reset; after: ${AFTER} singleton_present: ${SINGLETON}" + [[ "$SINGLETON" == "1" ]] || die "singleton row missing after reset — investigate before redeploy" + else + warn "skipped DB reset" + fi +fi + +# ============================================================================= +# Phase 4 — revert nico-core site_explorer bmc_proxy +# ============================================================================= +phase "Phase 4 — revert nico-core site_explorer config" +if $KEEP_NICO_CORE_CONFIG; then + warn "--keep-nico-core-config set; leaving bmc_proxy in place" +else + CM_JSON="$(mktemp)" + if kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" -o json > "$CM_JSON" 2>/dev/null; then + if grep -qE "bmc_proxy|override_target_(host|ip|port)" "$CM_JSON"; then + python3 - "$CM_JSON" <<'PY' +import json, sys +path = sys.argv[1] +cm = json.load(open(path)) +drop = ("bmc_proxy", "override_target_host", "override_target_ip", "override_target_port") +for k, v in cm["data"].items(): + lines = [ln for ln in v.splitlines() if not any(t in ln for t in drop)] + cm["data"][k] = "\n".join(lines) + ("\n" if v.endswith("\n") else "") +for f in ("resourceVersion","uid","creationTimestamp","managedFields"): + cm["metadata"].pop(f, None) +json.dump(cm, open(path, "w")) +PY + kubectl apply -f "$CM_JSON" >/dev/null + info "removed bmc_proxy / override_target_* lines; restarting nico-api" + kubectl rollout restart deployment/nico-api -n "$NICO_SYSTEM_NS" >/dev/null + kubectl rollout status deployment/nico-api -n "$NICO_SYSTEM_NS" --timeout=180s >/dev/null \ + || warn "nico-api rollout slow; continuing" + ok "site_explorer config reverted" + else + ok "no bmc_proxy / override_target_* present" + fi + else + warn "nico-api-site-config-files configmap not found; skipping" + fi +fi + +# ============================================================================= +# Phase 5 — remove site BMC root Vault credential +# ============================================================================= +phase "Phase 5 — remove machine-a-tron Vault credentials" +# site root + the host factory cred that setup-machine-a-tron.sh seeds. +# (The DPU factory + UEFI creds belong to nico-prereqs kvSeeds — left alone.) +# NB: vendor segment is lowercase ("dell") — BMCVendor's Display impl +# lowercases; must match what setup-machine-a-tron.sh seeds. +MAT_VAULT_PATHS="machines/bmc/site/root machines/all_hosts/factory_default/bmc-metadata-items/dell" +if $KEEP_VAULT_CRED; then + warn "--keep-vault-cred set; leaving credentials in place" +elif confirm "Delete Vault credentials seeded/rotated for machine-a-tron (site root, host factory, per-MAC)?"; then + for p in $MAT_VAULT_PATHS; do + if vault_cmd "vault kv get secrets/$p" >/dev/null 2>&1; then + vault_cmd "vault kv metadata delete secrets/$p" >/dev/null 2>&1 \ + || vault_cmd "vault kv delete secrets/$p" >/dev/null 2>&1 || true + ok "removed $p" + else + ok "already absent: $p" + fi + done + # Per-MAC rotated creds (machines/bmc//root) — written by site-explorer + # when it rotates each BMC's password to the site root. MUST be purged: a + # fresh mock boots at the factory password, but a surviving per-MAC entry + # tells site-explorer the BMC was already rotated, so it presents the old + # rotated password, gets 401 Unauthorized, and permanently latches + # AvoidLockout (NICO-SITEEXPLORER-144) on the endpoint. + # Batch server-side in ONE kubectl exec: at scale there are thousands of + # per-MAC entries and one exec per deletion takes HOURS (one round-trip + + # exec setup each); a shell loop on the vault pod does them all in seconds. + _n="$(vault_cmd 'count=0 +for m in $(vault kv list -format=yaml secrets/machines/bmc 2>/dev/null | sed "s/^- //" | grep -v "^site/"); do + vault kv metadata delete "secrets/machines/bmc/${m%/}/root" >/dev/null 2>&1 && count=$((count+1)) +done +echo $count' || echo 0)" + ok "removed ${_n:-0} per-MAC rotated creds (batched server-side)" +else + warn "kept credentials" +fi + +# ============================================================================= +# Phase 6 — verify clean state +# ============================================================================= +phase "Phase 6 — verify" +helm status "$RELEASE" -n "$MAT_NAMESPACE" >/dev/null 2>&1 && warn "release still present" || ok "release gone" +if ! $KEEP_DB; then + M="$(psql_q "SELECT count(*) FROM machines;" || echo '?')" + I="$(psql_q "SELECT count(*) FROM machine_interfaces;" || echo '?')" + info "machines=${M} machine_interfaces=${I} (expect 0/0)" +fi +phase "Done" +info "Cluster reset for a from-scratch machine-a-tron deploy." +info "Next: ./setup-machine-a-tron.sh (set REGISTRY_PULL_SECRET if you used --delete-namespace)" diff --git a/helm-prereqs/setup-machine-a-tron.sh b/helm-prereqs/setup-machine-a-tron.sh new file mode 100755 index 0000000000..de42b65f11 --- /dev/null +++ b/helm-prereqs/setup-machine-a-tron.sh @@ -0,0 +1,868 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================= +# setup-machine-a-tron.sh — deploy machine-a-tron end to end on a NICo site +# +# machine-a-tron is a bare-metal simulator: it hosts mock DPUs and servers via a +# Redfish BMC mock so NICo can run full ingestion flows without real hardware. +# This script sets up EVERYTHING needed for a running NICo Core site to discover +# and create the simulated machines, in setup.sh style (phased, idempotent). +# +# It assumes NICo Core (nico-api, postgres/nico-pg-cluster, Vault, ESO, +# cert-manager) is already deployed on the target cluster (i.e. setup.sh has run +# and the site is bootstrapped). It does NOT deploy NICo Core. +# +# --------------------------------------------------------------------------- +# WHY EACH STEP EXISTS — the non-obvious failure modes this script prevents +# (learned the hard way; do not remove without understanding them): +# +# * CA refresh (Phase 3): after a site reprovision the nico-system CA and +# nico-api are recreated. A machine-a-tron left over from before still trusts +# the OLD CA (stale nico-roots) and presents a client cert signed by the OLD +# CA, so mTLS to nico-api fails with "client error (Connect)" on every call. +# We always re-copy nico-roots from nico-system and (Phase 8) delete the old +# cert secret so cert-manager reissues from the CURRENT CA. +# +# * BMC site-root credential (Phase 5): site-explorer's check_preconditions +# requires the Vault credential machines/bmc/site/root. It is NOT in the +# default nico-prereqs kvSeeds, so without it site-explorer aborts every run +# with MissingCredentials and never explores anything. +# +# * bmc_proxy field name (Phase 6): the site_explorer config field is +# `bmc_proxy = "host:port"` (a single string). `override_target_host` is NOT +# a real field (older docs were wrong); override_target_ip/port are +# DEPRECATED. Setting bmc_proxy at launch also makes allow_changing_bmc_proxy +# default true, which is what lets machine-a-tron's configureBmcProxyHost +# runtime call succeed instead of being PermissionDenied. +# +# * expected_machines (chart value registerExpectedMachines: true): without a +# matching expected_machines row (by BMC MAC), MachineCreator refuses to +# create the managed host. machine-a-tron auto-registers them when the value +# is true — the script asserts it. +# +# * DHCP pool sizing (Phase 7): machine-a-tron needs one OOB IP per BMC — +# hostCount + hostCount*dpuPerHostCount. Overflowing the OOB pool yields +# "No IP addresses left in prefix ..." and machines never register. +# +# * machine_interfaces_deletion singleton (Phase 10 check): the +# machine_dhcp_records VIEW inner-joins the singleton row id=1. If it is ever +# deleted (e.g. by manual lease cleanup) the view returns zero rows and +# DiscoverDhcp fails with "no rows ... expected to return at least one row". +# The script restores the row if missing. NEVER delete it. To free stale +# leases, force-delete machine records via the admin CLI or reprovision — +# do NOT hand-delete interface/dhcp rows. +# +# --------------------------------------------------------------------------- +# Tool requirements: kubectl, helm, jq +# +# Required environment: +# KUBECONFIG Path to the target cluster kubeconfig (or current +# kubectl context already points at it). +# +# Optional environment: +# NICO_IMAGE_REGISTRY REQUIRED unless image.repository is set in the +# values file. Registry/repository prefix, without +# http(s):// (same convention as setup.sh). The +# machine-a-tron image is pulled from +# ${NICO_IMAGE_REGISTRY}/machine-a-tron. +# MAT_IMAGE_TAG REQUIRED unless image.tag is set in the values +# file. machine-a-tron image tag. +# REGISTRY_PULL_SECRET Registry password/API key. Only needed if the +# pull secret does not already exist in the +# machine-a-tron namespace. +# REGISTRY_PULL_USERNAME Username for the pull secret. Default: $oauthtoken +# MAT_NAMESPACE Deployment namespace. Default: nico-mat +# NICO_SYSTEM_NS NICo Core namespace. Default: nico-system +# POSTGRES_NS Postgres namespace. Default: postgres +# VAULT_NS Vault namespace. Default: vault +# BMC_USERNAME Site BMC root username. Default: root +# BMC_PASSWORD Site BMC root password (rotation target). MUST +# differ from the mock factory defaults. +# Default: NicoSiteRoot1 +# OOB_DHCP_RELAY OOB/underlay gateway (BMC DHCP relay). Auto-detected +# from nico-core site config if unset. +# ADMIN_DHCP_RELAY Admin network gateway. Auto-detected if unset. +# HOST_COUNT Override machines.dell-hosts.hostCount. +# DPU_PER_HOST Override machines.dell-hosts.dpuPerHostCount. +# CHART_DIR Path to the nico-machine-a-tron chart. +# Default: /helm/charts/nico-machine-a-tron +# VALUES_FILE Base values template. +# Default: /values/machine-a-tron.yaml +# +# Usage: +# export KUBECONFIG=/path/to/kubeconfig +# ./setup-machine-a-tron.sh # prompt before deploy +# ./setup-machine-a-tron.sh -y # non-interactive +# ./setup-machine-a-tron.sh --skip-nico-core-config # don't touch nico-core +# ============================================================================= + +set -euo pipefail + +# --- config / defaults ------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +MAT_NAMESPACE="${MAT_NAMESPACE:-nico-mat}" +NICO_SYSTEM_NS="${NICO_SYSTEM_NS:-nico-system}" +POSTGRES_NS="${POSTGRES_NS:-postgres}" +VAULT_NS="${VAULT_NS:-vault}" +BMC_USERNAME="${BMC_USERNAME:-root}" +# Site-wide BMC root password — the ROTATION TARGET. site-explorer logs into +# each mock BMC with its factory-default password, then rotates it to this +# value. It MUST DIFFER from both factory defaults below, or the rotation is a +# no-op and the mock keeps rejecting with "Factory-default password must be +# changed" (403) forever. +BMC_PASSWORD="${BMC_PASSWORD:-NicoSiteRoot1}" +# Factory defaults HARDCODED in the bmc-mock binary (crates/bmc-mock/src/lib.rs): +# host BMCs: DUMMY_FACTORY_PASSWORD = "factory_password" +# DPU BMCs: DUMMY_FACTORY_DPU_PASSWORD = "0penBmc" +# Do not change unless the mock changes. +FACTORY_HOST_BMC_PASSWORD="factory_password" +FACTORY_DPU_BMC_PASSWORD="0penBmc" +# Vendor path segment for the host factory cred. LOWERCASE is required: the +# credential path is built with format!("{vendor}") and BMCVendor's Display +# impl lowercases the variant name ("Dell Inc." → BMCVendor::Dell → "dell", +# crates/bmc-vendor/src/lib.rs impl Display). to_pascalcase() exists but is +# NOT used for Vault paths. +HOST_BMC_VENDOR="${HOST_BMC_VENDOR:-dell}" +# site-default UEFI passwords — check_preconditions requires them NON-EMPTY. +# The mock BMC does not validate them, so any non-empty value works. +UEFI_DPU_PASSWORD="${UEFI_DPU_PASSWORD:-bluefield}" +UEFI_HOST_PASSWORD="${UEFI_HOST_PASSWORD:-bluefield}" +REGISTRY_PULL_USERNAME="${REGISTRY_PULL_USERNAME:-\$oauthtoken}" +PULL_SECRET_NAME="${PULL_SECRET_NAME:-machine-a-tron-pull}" +RELEASE="nico-machine-a-tron" +BMC_MOCK_SVC="nico-machine-a-tron-bmc-mock" +BMC_MOCK_PORT="1266" +# site-explorer runs in nico-system, so it CANNOT resolve the bare service name +# (which resolves against its own namespace). bmc_proxy MUST use the +# cross-namespace FQDN of the bmc-mock service in the machine-a-tron namespace. +BMC_MOCK_FQDN="${BMC_MOCK_SVC}.${MAT_NAMESPACE}.svc.cluster.local" +NICO_DB="nico_system_nico" + +# --- deployment mode --------------------------------------------------------- +# override (default): all Redfish through site_explorer.bmc_proxy → one mock. +# scale: MetalLB/nginx mode — one LB Service per BMC, bmc_proxy UNSET, mock +# routes per-BMC via the Forwarded header. Uses values/machine-a-tron-scale.yaml +# plus two SIMULATED networks added to the nico-core site config (below) and +# raised site_explorer throughput knobs. See the chart README "METALLB MODE". +MAT_MODE="${MAT_MODE:-override}" +# Simulated networks for scale mode — must match the scale values file +# (relay addresses) and the MetalLB ipRange (inside the OOB prefix). +SCALE_OOB_PREFIX="10.100.0.0/17"; SCALE_OOB_GW="10.100.0.1" +SCALE_ADMIN_PREFIX="10.102.0.0/18"; SCALE_ADMIN_GW="10.102.0.1" +SCALE_RESERVE=1 +# site_explorer throughput knobs applied in scale mode (defaults 30/90/4 make +# 4500-host ingestion take ~9h; these bring it to ~1-2h). +SCALE_CONCURRENT_EXPLORATIONS="${SCALE_CONCURRENT_EXPLORATIONS:-100}" +# NB: keep explorations_per_run MODERATE. Identification and machine creation +# only run at the END of a completed explore_site cycle — a huge per-run value +# makes every cycle deep-scan hundreds of endpoints (dozens of Redfish calls +# each) and cycles stop completing, so machines are never created. ~120 keeps +# cycles under ~2 min while still sweeping the fleet quickly. +SCALE_EXPLORATIONS_PER_RUN="${SCALE_EXPLORATIONS_PER_RUN:-120}" +SCALE_MACHINES_CREATED_PER_RUN="${SCALE_MACHINES_CREATED_PER_RUN:-40}" + +CHART_DIR="${CHART_DIR:-${REPO_ROOT}/helm/charts/nico-machine-a-tron}" + +ASSUME_YES=false +SKIP_NICO_CORE_CONFIG=false +CM_JSON="" +MERGED_VALUES="" +cleanup() { rm -f "$CM_JSON" "$MERGED_VALUES" 2>/dev/null || true; } +trap cleanup EXIT + +for arg in "$@"; do + case "$arg" in + -y|--yes) ASSUME_YES=true ;; + --scale) MAT_MODE="scale" ;; + --skip-nico-core-config) SKIP_NICO_CORE_CONFIG=true ;; + -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//' | head -110; exit 0 ;; + *) echo "Unknown argument: $arg" >&2; exit 2 ;; + esac +done + +if [[ "$MAT_MODE" == "scale" ]]; then + VALUES_FILE="${VALUES_FILE:-${SCRIPT_DIR}/values/machine-a-tron-scale.yaml}" +else + VALUES_FILE="${VALUES_FILE:-${SCRIPT_DIR}/values/machine-a-tron.yaml}" +fi + +# --- helpers ----------------------------------------------------------------- +_c() { printf '\033[%sm' "$1"; } +BOLD="$(_c 1)"; RED="$(_c 31)"; GREEN="$(_c 32)"; YEL="$(_c 33)"; BLU="$(_c 34)"; NC="$(_c 0)" +phase() { echo; echo "${BOLD}${BLU}== $* ==${NC}"; } +info() { echo " $*"; } +ok() { echo " ${GREEN}✓${NC} $*"; } +warn() { echo " ${YEL}!${NC} $*" >&2; } +die() { echo "${RED}ERROR:${NC} $*" >&2; exit 1; } +confirm() { + $ASSUME_YES && return 0 + read -r -p " $* [y/N] " ans + [[ "$ans" == "y" || "$ans" == "Y" ]] +} + +# psql against the consolidated NICo DB on the Patroni primary +PG_PRIMARY="" +_pg_primary() { + [[ -n "$PG_PRIMARY" ]] && { echo "$PG_PRIMARY"; return; } + PG_PRIMARY="$(kubectl get pods -n "$POSTGRES_NS" -l application=spilo \ + -o jsonpath='{range .items[*]}{.metadata.name} {.metadata.labels.spilo-role}{"\n"}{end}' 2>/dev/null \ + | awk '$2=="master"{print $1}' | head -1)" + echo "$PG_PRIMARY" +} +psql_q() { + local pg; pg="$(_pg_primary)" + [[ -n "$pg" ]] || die "no Patroni primary found in namespace $POSTGRES_NS" + kubectl exec -n "$POSTGRES_NS" "$pg" -- su postgres -c "psql -d $NICO_DB -v ON_ERROR_STOP=1 -tAc \"$1\"" 2>/dev/null +} +# count query that always yields a number — a transient kubectl/psql failure +# returns "0" instead of an empty string that would blow up (( )) arithmetic. +psql_count() { local r; r="$(psql_q "$1" || true)"; echo "${r:-0}"; } +# vault CLI on vault-0 using the root token stored in nico-system/nico-vault-token. +# Token is cached after the first read (it does not change within a run). +# env vars are exported (not inline-prefixed) so they apply across pipes, e.g. +# `echo ... | vault kv put ... -` — an inline prefix would bind them to echo only. +_VAULT_TOKEN="" +vault_cmd() { + if [[ -z "$_VAULT_TOKEN" ]]; then + _VAULT_TOKEN="$(kubectl get secret nico-vault-token -n "$NICO_SYSTEM_NS" -o jsonpath='{.data.token}' | base64 -d)" + [[ -n "$_VAULT_TOKEN" ]] || die "could not read nico-vault-token from $NICO_SYSTEM_NS" + fi + kubectl exec -n "$VAULT_NS" vault-0 -c vault -- sh -c \ + "export VAULT_TOKEN='$_VAULT_TOKEN' VAULT_ADDR=https://127.0.0.1:8200 VAULT_SKIP_VERIFY=true; $1" 2>/dev/null +} +# copy a secret from nico-system into the machine-a-tron namespace (strip metadata) +copy_secret() { + local name="$1" + kubectl get secret "$name" -n "$NICO_SYSTEM_NS" -o json 2>/dev/null \ + | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid,.metadata.creationTimestamp,.metadata.ownerReferences,.metadata.annotations,.metadata.managedFields)' \ + | kubectl apply -n "$MAT_NAMESPACE" -f - >/dev/null +} + +# ============================================================================= +# Phase 0 — preflight +# ============================================================================= +phase "Phase 0 — preflight" +for t in kubectl helm jq; do command -v "$t" >/dev/null || die "$t not found in PATH"; done +kubectl version -o json >/dev/null 2>&1 || kubectl cluster-info >/dev/null 2>&1 || die "cannot reach the cluster (check KUBECONFIG)" +ok "tools present, cluster reachable" +[[ -d "$CHART_DIR" ]] || die "chart dir not found: $CHART_DIR" +[[ -f "$VALUES_FILE" ]] || die "values file not found: $VALUES_FILE" +kubectl get deploy nico-api -n "$NICO_SYSTEM_NS" >/dev/null 2>&1 || die "nico-api not found in $NICO_SYSTEM_NS — deploy NICo Core (setup.sh) first" +[[ -n "$(_pg_primary)" ]] || die "no Postgres primary in $POSTGRES_NS" +kubectl get pod vault-0 -n "$VAULT_NS" >/dev/null 2>&1 || die "vault-0 not found in $VAULT_NS" +ok "NICo Core present: nico-api, postgres primary $(_pg_primary), vault-0" + +# portable extraction (macOS BSD sed/grep lack \s): [[:space:]] + awk on quotes +MAT_IMAGE_TAG="${MAT_IMAGE_TAG:-$(grep -E '^[[:space:]]*tag:' "$VALUES_FILE" | head -1 | awk -F'"' '{print $2}')}" +MAT_IMAGE_REPO="${MAT_IMAGE_REPO:-$(grep -E '^[[:space:]]*repository:' "$VALUES_FILE" | head -1 | awk -F'"' '{print $2}')}" +# Registry-agnostic (mirrors setup.sh): the image location comes from the +# environment, never from committed defaults. +if [[ -z "$MAT_IMAGE_REPO" ]]; then + [[ -n "${NICO_IMAGE_REGISTRY:-}" ]] || die "NICO_IMAGE_REGISTRY is unset and the values file has no image.repository (see setup.sh conventions)" + MAT_IMAGE_REPO="${NICO_IMAGE_REGISTRY}/machine-a-tron" +fi +HOST_COUNT="${HOST_COUNT:-$(grep -E '^[[:space:]]*hostCount:' "$VALUES_FILE" | head -1 | grep -oE '[0-9]+')}" +DPU_PER_HOST="${DPU_PER_HOST:-$(grep -E '^[[:space:]]*dpuPerHostCount:' "$VALUES_FILE" | head -1 | grep -oE '[0-9]+')}" +[[ -n "$MAT_IMAGE_TAG" ]] || die "MAT_IMAGE_TAG is unset and the values file has no image.tag" +[[ "$HOST_COUNT" =~ ^[0-9]+$ && "$DPU_PER_HOST" =~ ^[0-9]+$ ]] \ + || die "could not determine hostCount/dpuPerHostCount from $VALUES_FILE (set HOST_COUNT / DPU_PER_HOST)" +# Passwords are inlined into sh -c JSON heredocs on the vault pod; quotes, +# backslashes, or whitespace would break quoting or corrupt the JSON silently. +for _pw in "$BMC_PASSWORD" "$UEFI_DPU_PASSWORD" "$UEFI_HOST_PASSWORD"; do + case "$_pw" in + *[\'\"\\\ ]*) die "passwords must not contain quotes, backslashes, or spaces (BMC_PASSWORD / UEFI_*_PASSWORD)" ;; + esac +done +info "image: ${MAT_IMAGE_REPO}:${MAT_IMAGE_TAG} hosts: ${HOST_COUNT} dpus/host: ${DPU_PER_HOST}" + +# ============================================================================= +# Phase 1 — namespace +# ============================================================================= +phase "Phase 1 — namespace ${MAT_NAMESPACE}" +kubectl create namespace "$MAT_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +# label so ESO / nico-roots sync treats it as a managed namespace +kubectl label namespace "$MAT_NAMESPACE" nico.nvidia.com/managed=true --overwrite >/dev/null +ok "namespace ready" + +# ============================================================================= +# Phase 2 — image pull secret +# ============================================================================= +phase "Phase 2 — image pull secret" +if kubectl get secret "$PULL_SECRET_NAME" -n "$MAT_NAMESPACE" >/dev/null 2>&1; then + ok "pull secret ${PULL_SECRET_NAME} already exists" +elif [[ -n "${REGISTRY_PULL_SECRET:-}" ]]; then + kubectl create secret docker-registry "$PULL_SECRET_NAME" -n "$MAT_NAMESPACE" \ + --docker-server="${MAT_IMAGE_REPO%%/*}" --docker-username="$REGISTRY_PULL_USERNAME" \ + --docker-password="$REGISTRY_PULL_SECRET" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null + ok "pull secret ${PULL_SECRET_NAME} created" +else + warn "pull secret ${PULL_SECRET_NAME} missing and REGISTRY_PULL_SECRET unset" + warn " → set REGISTRY_PULL_SECRET, or ensure the image is already cached on nodes" +fi + +# ============================================================================= +# Phase 3 — refresh CA + Vault secrets from nico-system (GOTCHA: stale CA) +# ============================================================================= +phase "Phase 3 — refresh nico-roots CA + Vault secrets" +copy_secret nico-roots +ok "nico-roots synced from ${NICO_SYSTEM_NS} (current CA)" +for s in nico-vault-approle-tokens nico-vault-token; do + if kubectl get secret "$s" -n "$NICO_SYSTEM_NS" >/dev/null 2>&1; then + copy_secret "$s"; ok "$s synced" + else + warn "$s not present in ${NICO_SYSTEM_NS}; skipping (may produce log noise only)" + fi +done + +# ============================================================================= +# Phase 4 — seed site BMC root credential (GOTCHA: not in default kvSeeds) +# ============================================================================= +phase "Phase 4 — seed Vault BMC + UEFI credentials" +_kv_put() { # $1=path $2=username $3=password + vault_cmd "echo '{\"UsernamePassword\":{\"username\":\"$2\",\"password\":\"$3\"}}' | vault kv put secrets/$1 -" >/dev/null \ + || die "failed to write $1 to Vault" +} +_kv_password() { # $1=path → prints current password (empty if absent) + # `|| true` is load-bearing: when the path is absent, vault kv get fails and + # under `set -e` a failing $(...) assignment would kill the whole script. + vault_cmd "vault kv get -format=json secrets/$1" 2>/dev/null | jq -r '.data.data.UsernamePassword.password // empty' 2>/dev/null || true +} +# Factory-default creds — the INITIAL login site-explorer uses before rotating. +# Host and DPU factories DIFFER (see constants above); the wrong one yields 401 +# Unauthorized on every host BMC and a permanent AvoidLockout latch. +_kv_put "machines/all_hosts/factory_default/bmc-metadata-items/${HOST_BMC_VENDOR}" root "$FACTORY_HOST_BMC_PASSWORD" +ok "factory host cred: .../${HOST_BMC_VENDOR} = root/${FACTORY_HOST_BMC_PASSWORD}" +_kv_put "machines/all_dpus/factory_default/bmc-metadata-items/root" root "$FACTORY_DPU_BMC_PASSWORD" +ok "factory DPU cred: .../root = root/${FACTORY_DPU_BMC_PASSWORD}" +# Site-wide root — the rotation target; must differ from both factory passwords. +_cur="$(_kv_password machines/bmc/site/root)" +if [[ -z "$_cur" || "$_cur" == "$FACTORY_HOST_BMC_PASSWORD" || "$_cur" == "$FACTORY_DPU_BMC_PASSWORD" ]]; then + _kv_put "machines/bmc/site/root" "$BMC_USERNAME" "$BMC_PASSWORD" + ok "machines/bmc/site/root seeded (${BMC_USERNAME}/**** — distinct from factory)" +else + ok "machines/bmc/site/root already present with a non-factory password" +fi +# Self-heal stale per-MAC rotated creds (machines/bmc//root). site-explorer +# writes one per BMC after rotating its password to the site root; entries +# surviving from a previous deployment poison a fresh one — the fresh mock is at +# the factory password, but the per-MAC entry makes site-explorer present the +# old rotated password: 401 → permanent AvoidLockout. Only safe to purge when +# the machine graph is empty (live machines' per-MAC creds are real). +# "Fresh deployment" = no machines AND no interfaces. machines==0 alone is +# NOT enough: mid-ingestion (interfaces DHCP'd, endpoints explored, machines +# not yet created) the per-MAC creds are LIVE — purging them forces every +# endpoint back through the credential fallback. +if [[ "$(psql_count "SELECT count(*) FROM machines;")" == "0" \ + && "$(psql_count "SELECT count(*) FROM machine_interfaces;")" == "0" ]]; then + # Batched server-side in ONE kubectl exec — thousands of entries at scale; + # one exec per deletion takes hours. + _n="$(vault_cmd 'count=0 +for m in $(vault kv list -format=yaml secrets/machines/bmc 2>/dev/null | sed "s/^- //" | grep -v "^site/"); do + vault kv metadata delete "secrets/machines/bmc/${m%/}/root" >/dev/null 2>&1 && count=$((count+1)) +done +echo $count' || echo 0)" + if [[ "${_n:-0}" != "0" ]]; then + warn "purged ${_n} stale per-MAC BMC creds from a previous deployment (machine graph was empty)" + fi +fi +# site-explorer's check_preconditions ALSO requires the DPU + Host site_default +# UEFI creds to have a NON-EMPTY password. The nico-prereqs kvSeeds create these +# with empty passwords ("SITE SECRET: populate per site"), which fails the check +# with "vault does not have a valid password entry". Seed a non-empty value if +# the current password is empty/absent (the mock BMC does not validate it). +_seed_uefi() { # $1=vault path $2=password + local path="$1" pw="$2" cur + cur="$(_kv_password "$path")" + if [[ -n "$cur" ]]; then + ok "precondition cred present + valid: $path" + else + vault_cmd "echo '{\"UsernamePassword\":{\"username\":\"admin\",\"password\":\"${pw}\"}}' | vault kv put secrets/$path -" >/dev/null \ + || die "failed to seed UEFI cred $path" + ok "seeded UEFI cred (was empty): $path" + fi +} +_seed_uefi "machines/all_dpus/site_default/uefi-metadata-items/auth" "$UEFI_DPU_PASSWORD" +_seed_uefi "machines/all_hosts/site_default/uefi-metadata-items/auth" "$UEFI_HOST_PASSWORD" + +# ============================================================================= +# Phase 5 — configure nico-core site config for the selected mode +# override: set site_explorer.bmc_proxy (GOTCHA: field name; FQDN required) +# scale: REMOVE bmc_proxy, add simulated networks + throughput knobs +# ============================================================================= +phase "Phase 5 — nico-core site config (${MAT_MODE} mode)" +if $SKIP_NICO_CORE_CONFIG; then + warn "--skip-nico-core-config set; configure [site_explorer]/[networks] manually for ${MAT_MODE} mode" +elif [[ "$MAT_MODE" == "scale" ]]; then + CM_JSON="$(mktemp)" + kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" -o json > "$CM_JSON" 2>/dev/null \ + || die "nico-api-site-config-files configmap not found" + _PATCH_RESULT="$(SCALE_OOB_PREFIX="$SCALE_OOB_PREFIX" SCALE_OOB_GW="$SCALE_OOB_GW" \ + SCALE_ADMIN_PREFIX="$SCALE_ADMIN_PREFIX" SCALE_ADMIN_GW="$SCALE_ADMIN_GW" \ + SCALE_RESERVE="$SCALE_RESERVE" \ + BMC_PROXY="${BMC_MOCK_FQDN}:${BMC_MOCK_PORT}" \ + KNOB_CONC="$SCALE_CONCURRENT_EXPLORATIONS" KNOB_EPR="$SCALE_EXPLORATIONS_PER_RUN" \ + KNOB_MCPR="$SCALE_MACHINES_CREATED_PER_RUN" python3 - "$CM_JSON" <<'PY' +import json, os, sys +path = sys.argv[1] +cm = json.load(open(path)) +env = os.environ +# lines managed by this script inside [site_explorer] +drop = ("bmc_proxy", "override_target_host", "override_target_ip", "override_target_port", + "concurrent_explorations", "explorations_per_run", "machines_created_per_run") +knobs = [ + # PROXY-DIRECT: the Redfish client injects "Forwarded: host=" + # whenever bmc_proxy is set; the mock's registry routes on it. One + # ClusterIP service serves the whole simulated fleet. + f' bmc_proxy = "{env["BMC_PROXY"]}"', + f' concurrent_explorations = {env["KNOB_CONC"]}', + f' explorations_per_run = {env["KNOB_EPR"]}', + f' machines_created_per_run = {env["KNOB_MCPR"]}', +] +networks = f''' +# --- simulated networks for machine-a-tron scale testing (managed by +# --- setup-machine-a-tron.sh --scale; safe to leave in place) --- +[networks.simulated-oob] +type = "underlay" +prefix = "{env["SCALE_OOB_PREFIX"]}" +gateway = "{env["SCALE_OOB_GW"]}" +mtu = 9000 +reserve_first = {env["SCALE_RESERVE"]} + +[networks.simulated-admin] +type = "admin" +prefix = "{env["SCALE_ADMIN_PREFIX"]}" +gateway = "{env["SCALE_ADMIN_GW"]}" +mtu = 9000 +reserve_first = {env["SCALE_RESERVE"]} +''' +# Machine creation allocates one loopback IP per machine from pools.lo-ip — +# site templates ship tiny ranges (dev6: 3 addresses) that exhaust instantly +# ("Resource pool lo-ip is empty"). Pools DO reconcile at startup (unlike +# networks), so appending a simulated range takes effect on restart. +SIM_LO = ', { start = "10.103.0.1", end = "10.103.63.254" }]' +changed = False +for k, v in cm["data"].items(): + if "[site_explorer]" not in v: + continue + out, in_lo = [], False + for ln in v.splitlines(): + s = ln.strip() + if any(t in ln for t in drop): + continue + if s.startswith("[pools."): + in_lo = (s == "[pools.lo-ip]") + if in_lo and s.startswith("ranges") and "10.103.0.1" not in ln: + r = ln.rstrip(); idx = r.rfind("]") + ln = r[:idx] + SIM_LO + r[idx+1:] + out.append(ln) + if s == "[site_explorer]": + out.extend(knobs) + new = "\n".join(out) + ("\n" if v.endswith("\n") else "") + if "[networks.simulated-oob]" not in new: + new = new.rstrip("\n") + "\n" + networks + if new != v: + cm["data"][k] = new + changed = True +for f in ("resourceVersion","uid","creationTimestamp","managedFields"): + cm["metadata"].pop(f, None) +json.dump(cm, open(path, "w")) +print("changed" if changed else "nochange") +PY +)" + if [[ "$_PATCH_RESULT" == "changed" ]]; then + kubectl apply -f "$CM_JSON" >/dev/null + info "scale config applied (proxy-direct bmc_proxy, simulated networks, knobs, lo-ip); restarting nico-api" + kubectl rollout restart deployment/nico-api -n "$NICO_SYSTEM_NS" >/dev/null + kubectl rollout status deployment/nico-api -n "$NICO_SYSTEM_NS" --timeout=180s >/dev/null \ + || warn "nico-api rollout did not complete in time; continuing" + ok "scale networks: oob ${SCALE_OOB_PREFIX} (gw ${SCALE_OOB_GW}), admin ${SCALE_ADMIN_PREFIX} (gw ${SCALE_ADMIN_GW})" + ok "site_explorer knobs: concurrent=${SCALE_CONCURRENT_EXPLORATIONS} per_run=${SCALE_EXPLORATIONS_PER_RUN} create/run=${SCALE_MACHINES_CREATED_PER_RUN}" + else + ok "scale config already in place" + fi + + # --- ensure the simulated network SEGMENTS exist ------------------------- + # Config-driven segment creation (create_initial_networks) is bootstrap- + # once: it SKIPS entirely when the DB has multiple DNS domains ("we + # probably created the network much earlier", crates/api-core/src/db_init.rs). + # On an established site the new [networks.*] stanzas therefore never + # materialize and every mat DHCP fails with "No network segment defined + # for relay addresses". Fallback: clone an existing segment of the same + # type (identity fields overridden, vlan/vni cleared) + insert the prefix. + _ensure_segment() { # $1=name $2=type-ilike $3=prefix $4=gateway $5=reserve + local name="$1" typ="$2" pfx="$3" gw="$4" rsv="$5" + if [[ "$(psql_count "SELECT count(*) FROM network_segments WHERE name='${name}';")" != "0" ]]; then + ok "segment ${name} present" + return + fi + warn "segment ${name} missing (config seeding is bootstrap-once on multi-domain sites) — creating from template" + # allocation_strategy is forced to 'dynamic': templates may be + # 'reserved' (static-assignments segments), which rejects every mat + # DHCP with "configured for static DHCP leases only". + psql_q "INSERT INTO network_segments + SELECT (jsonb_populate_record(ns, jsonb_build_object( + 'id', gen_random_uuid()::text, 'name', '${name}', + 'allocation_strategy', 'dynamic', + 'vlan_id', NULL, 'vni_id', NULL))).* + FROM network_segments ns + WHERE ns.network_segment_type::text ILIKE '${typ}' LIMIT 1;" >/dev/null \ + || die "failed to create segment ${name} (no ${typ} template segment?)" + psql_q "INSERT INTO network_prefixes (segment_id, prefix, gateway, num_reserved) + SELECT id, '${pfx}'::cidr, '${gw}'::inet, ${rsv} + FROM network_segments WHERE name='${name}';" >/dev/null \ + || die "failed to add prefix ${pfx} to segment ${name}" + ok "segment ${name} created: ${pfx} (gw ${gw}, reserved ${rsv})" + } + _ensure_segment "simulated-oob" "underlay" "$SCALE_OOB_PREFIX" "$SCALE_OOB_GW" "$SCALE_RESERVE" + _ensure_segment "simulated-admin" "admin" "$SCALE_ADMIN_PREFIX" "$SCALE_ADMIN_GW" "$SCALE_RESERVE" + + # --- widen the lo-ip resource pool --------------------------------------- + # Machine creation allocates one loopback IP per machine from resource_pool + # rows. Pool DEFINITIONS are seed-once ("Declaration has drifted since + # seed ... not re-applying", crates/api-db/src/resource_pool.rs), so config + # changes to [pools.lo-ip] are IGNORED on established sites — rows must be + # inserted directly. Site templates ship tiny ranges (dev6: 3 addresses) + # that exhaust instantly ("Resource pool lo-ip is empty"). + _LO_FREE="$(psql_count "SELECT count(*) FROM resource_pool WHERE name='lo-ip' AND allocated IS NULL;")" + if (( _LO_FREE < HOST_COUNT * (1 + DPU_PER_HOST) )); then + info "widening lo-ip pool (${_LO_FREE} free < needed) with simulated range 10.103.0.1-10.103.63.254" + psql_q "INSERT INTO resource_pool (name, value, value_type, auto_assign, state, state_version, created) + SELECT 'lo-ip', host('10.103.0.0'::inet + g), 'ipv4', true, + '{\\\"state\\\":\\\"free\\\"}'::jsonb, + (SELECT state_version FROM resource_pool WHERE name='lo-ip' LIMIT 1), + now() + FROM generate_series(1, 16382) g + WHERE NOT EXISTS (SELECT 1 FROM resource_pool rp WHERE rp.name='lo-ip' AND rp.value = host('10.103.0.0'::inet + g));" >/dev/null \ + || die "failed to widen lo-ip pool" + ok "lo-ip pool: $(psql_count "SELECT count(*) FROM resource_pool WHERE name='lo-ip' AND allocated IS NULL;") free" + else + ok "lo-ip pool has ${_LO_FREE} free addresses" + fi +else + CM_JSON="$(mktemp)" + kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" -o json > "$CM_JSON" 2>/dev/null \ + || die "nico-api-site-config-files configmap not found" + if grep -q "bmc_proxy = \"${BMC_MOCK_FQDN}:${BMC_MOCK_PORT}\"" "$CM_JSON"; then + ok "bmc_proxy already configured" + else + _PATCH_RESULT="$(BMC_PROXY="${BMC_MOCK_FQDN}:${BMC_MOCK_PORT}" python3 - "$CM_JSON" <<'PY' +import json, os, sys +proxy = os.environ["BMC_PROXY"] +path = sys.argv[1] +cm = json.load(open(path)) +line = f' bmc_proxy = "{proxy}"' +drop = ("bmc_proxy", "override_target_host", "override_target_ip", "override_target_port") +changed = False +for k, v in cm["data"].items(): + if "[site_explorer]" not in v: + continue + out = [] + for ln in v.splitlines(): + if any(t in ln for t in drop): # strip any stale/legacy proxy lines + continue + out.append(ln) + if ln.strip() == "[site_explorer]": + out.append(line) # insert the correct FQDN bmc_proxy + cm["data"][k] = "\n".join(out) + ("\n" if v.endswith("\n") else "") + changed = True +for f in ("resourceVersion","uid","creationTimestamp","managedFields"): + cm["metadata"].pop(f, None) +json.dump(cm, open(path, "w")) +print("changed" if changed else "nochange") +PY +)" + if [[ "$_PATCH_RESULT" == "changed" ]]; then + kubectl apply -f "$CM_JSON" >/dev/null + info "configmap patched; restarting nico-api to load bmc_proxy" + kubectl rollout restart deployment/nico-api -n "$NICO_SYSTEM_NS" >/dev/null + kubectl rollout status deployment/nico-api -n "$NICO_SYSTEM_NS" --timeout=180s >/dev/null \ + || warn "nico-api rollout did not complete in time; continuing" + ok "bmc_proxy set to ${BMC_MOCK_FQDN}:${BMC_MOCK_PORT}" + else + ok "no [site_explorer] section found to patch — check the configmap manually" + fi + fi +fi + +# ============================================================================= +# Phase 6 — resolve DHCP relays + sizing check +# ============================================================================= +phase "Phase 6 — DHCP relays + pool sizing (${MAT_MODE})" +if [[ "$MAT_MODE" == "scale" ]]; then + # scale mode uses the SIMULATED networks added in Phase 5 — constants, + # no live-config parsing needed. + OOB_PREFIX="$SCALE_OOB_PREFIX"; OOB_DHCP_RELAY="${OOB_DHCP_RELAY:-$SCALE_OOB_GW}" + ADMIN_PREFIX="$SCALE_ADMIN_PREFIX"; ADMIN_DHCP_RELAY="${ADMIN_DHCP_RELAY:-$SCALE_ADMIN_GW}" + OOB_RESERVE="$SCALE_RESERVE"; ADMIN_RESERVE="$SCALE_RESERVE" +else + SITE_CFG="$(kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" \ + -o jsonpath='{.data.nico-api-site-config\.toml}' 2>/dev/null || true)" + # older deployments carry the config under the carbide-* key only + [[ -n "$SITE_CFG" ]] || SITE_CFG="$(kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" \ + -o jsonpath='{.data.carbide-api-site-config\.toml}' 2>/dev/null || true)" + # gateway lines appear as: gateway = "10.x.y.z" (admin then underlay in template order). + # Portable parse (no mapfile / negative index — macOS ships bash 3.2). + # Comment lines are stripped first: the config template carries commented + # examples (e.g. "# reserve_first = 5") that would otherwise be picked up. + SITE_CFG_CODE="$(printf '%s\n' "$SITE_CFG" | grep -vE '^[[:space:]]*#' || true)" + GW_LIST="$(printf '%s\n' "$SITE_CFG_CODE" | grep -oE 'gateway = "[0-9.]+"' | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' || true)" + PFX_LIST="$(printf '%s\n' "$SITE_CFG_CODE" | grep -oE 'prefix = "[0-9./]+"' | grep -oE '[0-9./]+' || true)" + RSV_LIST="$(printf '%s\n' "$SITE_CFG_CODE" | grep -oE 'reserve_first = [0-9]+' | grep -oE '[0-9]+' || true)" + # admin segment is the FIRST prefix/gateway pair, OOB/underlay the LAST. + ADMIN_PREFIX="$(printf '%s\n' "$PFX_LIST" | head -1)" + OOB_PREFIX="$(printf '%s\n' "$PFX_LIST" | tail -1)" + OOB_DHCP_RELAY="${OOB_DHCP_RELAY:-$(printf '%s\n' "$GW_LIST" | tail -1)}" + ADMIN_DHCP_RELAY="${ADMIN_DHCP_RELAY:-$(printf '%s\n' "$GW_LIST" | head -1)}" + # Usable IPs per pool = 2^(32-mask) − reserve_first − 1 (broadcast). + # reserve_first covers the network address, gateway, and operator-reserved + # leading addresses — verified live: /28 with reserve_first=5 → 10 usable. + ADMIN_RESERVE="$(printf '%s\n' "$RSV_LIST" | head -1)"; ADMIN_RESERVE="${ADMIN_RESERVE:-5}" + OOB_RESERVE="$(printf '%s\n' "$RSV_LIST" | tail -1)"; OOB_RESERVE="${OOB_RESERVE:-5}" +fi +[[ -n "$OOB_DHCP_RELAY" && -n "$ADMIN_DHCP_RELAY" ]] \ + || die "could not resolve DHCP relays; set OOB_DHCP_RELAY and ADMIN_DHCP_RELAY" +ok "OOB: relay ${OOB_DHCP_RELAY} prefix ${OOB_PREFIX:-unknown}" +ok "admin: relay ${ADMIN_DHCP_RELAY} prefix ${ADMIN_PREFIX:-unknown}" +_usable() { local m="${1##*/}" r="$2"; local u=$(( (1 << (32 - m)) - r - 1 )); (( u < 0 )) && u=0; echo "$u"; } +# Demand per pool (measured live): +# OOB = hostCount*(1 + dpuPerHost) — one BMC IP per host and per DPU +# admin = hostCount*(dpuPerHost + 1) — one host-PF IP per DPU at DHCP time, +# PLUS one admin IP per host allocated by machine creation (creation +# fails with "No IP addresses left in prefix " without it) +if [[ "${OOB_PREFIX:-}" == */* && "${ADMIN_PREFIX:-}" == */* ]]; then + OOB_USABLE="$(_usable "$OOB_PREFIX" "$OOB_RESERVE")"; ADMIN_USABLE="$(_usable "$ADMIN_PREFIX" "$ADMIN_RESERVE")" + # max hosts each pool supports, then take the min + FIT_OOB=$(( OOB_USABLE / (1 + DPU_PER_HOST) )) + FIT_ADMIN=$(( ADMIN_USABLE / (DPU_PER_HOST + 1) )) + FIT=$(( FIT_OOB < FIT_ADMIN ? FIT_OOB : FIT_ADMIN )) + info "pool fit: OOB ${OOB_PREFIX} ≈${OOB_USABLE} usable → ≤${FIT_OOB} hosts; admin ${ADMIN_PREFIX} ≈${ADMIN_USABLE} usable → ≤${FIT_ADMIN} hosts" + if (( HOST_COUNT > FIT )); then + (( FIT < 1 )) && die "pools too small for even 1 host × ${DPU_PER_HOST} DPUs — widen the admin/OOB prefixes or lower DPU_PER_HOST" + warn "requested ${HOST_COUNT} hosts exceeds pool capacity (${FIT}) — auto-fitting hostCount=${FIT}" + warn " (override with HOST_COUNT/DPU_PER_HOST env vars, or widen the site's DHCP prefixes)" + confirm "Proceed with hostCount=${FIT} × ${DPU_PER_HOST} DPUs?" || die "aborted on sizing" + HOST_COUNT="$FIT" + fi + NEED=$(( HOST_COUNT + HOST_COUNT * DPU_PER_HOST )) + ok "sizing: ${HOST_COUNT} hosts × ${DPU_PER_HOST} DPUs → ${NEED} OOB + $(( HOST_COUNT * (DPU_PER_HOST + 1) )) admin IPs" +else + NEED=$(( HOST_COUNT + HOST_COUNT * DPU_PER_HOST )) + warn "could not parse both pool prefixes — skipping sizing check (need ${NEED} OOB IPs)" +fi + +# ============================================================================= +# Phase 7 — DB safety: restore machine_interfaces_deletion singleton (GOTCHA) +# ============================================================================= +phase "Phase 7 — DB safety checks" +# "!= 1" (not "== 0") so a transient query failure (empty result) also takes +# the restore path — the INSERT is idempotent (ON CONFLICT DO NOTHING). +SINGLETON="$(psql_count "SELECT count(*) FROM machine_interfaces_deletion WHERE id=1;")" +if [[ "$SINGLETON" != "1" ]]; then + warn "machine_interfaces_deletion singleton (id=1) missing — restoring" + warn " (its absence breaks the machine_dhcp_records view → DiscoverDhcp 'no rows' errors)" + psql_q "INSERT INTO machine_interfaces_deletion (id) VALUES (1) ON CONFLICT (id) DO NOTHING;" >/dev/null + ok "singleton restored" +else + ok "machine_interfaces_deletion singleton present" +fi +ORPHANS="$(psql_count "SELECT count(*) FROM machine_interfaces mi WHERE NOT EXISTS (SELECT 1 FROM machines m WHERE m.id = mi.machine_id);")" +MACHINES_NOW="$(psql_count "SELECT count(*) FROM machines;")" +if [[ "${ORPHANS:-0}" -gt 0 && "${MACHINES_NOW:-0}" == "0" ]]; then + warn "${ORPHANS} orphaned machine_interfaces (no parent machine) may hold OOB leases" + warn " → if DHCP later reports exhaustion, force-delete stale records via the admin CLI" + warn " or reprovision. Do NOT hand-delete interface/dhcp rows (breaks the singleton)." +fi + +# ============================================================================= +# Phase 8 — reissue client cert from current CA (GOTCHA: stale cert) +# ============================================================================= +phase "Phase 8 — reissue machine-a-tron client cert" +# Always delete: a cert issued under a previous CA fails mTLS to nico-api with +# "client error (Connect)" on every call. cert-manager reissues from the +# current CA within seconds of the deploy — there is no reason to keep it. +kubectl delete secret "${RELEASE}-certificate" -n "$MAT_NAMESPACE" --ignore-not-found >/dev/null 2>&1 +ok "client cert cleared; cert-manager reissues from the current CA on deploy" + +# ============================================================================= +# Phase 9 — deploy the chart +# ============================================================================= +phase "Phase 9 — helm upgrade --install ${RELEASE}" +MERGED_VALUES="$(mktemp)" +# Site-specific overrides ONLY (never committed). Passed as a second -f so Helm +# deep-merges it over the base template (last -f wins per key) — avoids +# unreliable duplicate top-level keys within a single YAML file. +cat > "$MERGED_VALUES" <> "$MERGED_VALUES" </dev/null 2>&1 \ + && ok "client certificate Ready" || warn "certificate not Ready yet — check cert-manager" + +# wait windows scale with the deployment size (scale mode: hundreds-thousands) +IFACE_WAIT=$(( 90 + NEED )); (( IFACE_WAIT > 1800 )) && IFACE_WAIT=1800 +info "giving machine-a-tron time to register + DHCP (up to ${IFACE_WAIT}s)..." +_end=$((SECONDS+IFACE_WAIT)) +IFACES=0 +while (( SECONDS < _end )); do + IFACES="$(psql_count "SELECT count(*) FROM machine_interfaces;")" + (( IFACES >= NEED )) && break + sleep 10 +done +IPS="$(psql_count "SELECT count(*) FROM machine_interface_addresses;")" +info "machine_interfaces=${IFACES} (need ${NEED}) ips_allocated=${IPS}" +(( IFACES >= NEED )) && ok "BMC interfaces registered + DHCP allocated" \ + || warn "fewer interfaces than expected — check pool sizing / bmc DHCP" + +# --- expected_machines: required for machine creation (matched by BMC MAC) --- +# machine-a-tron auto-registers them (registerExpectedMachines: true), but on +# nico-api builds without the Machineatron→AddExpectedMachine RBAC grant +# (crates/api-core/src/auth/internal_rbac_rules.rs) the call is 403'd. Fall +# back to direct DB registration mirroring what the API call would create. +info "waiting for expected_machines (auto-registration)..." +_end=$((SECONDS+45)); EXPECTED=0 +while (( SECONDS < _end )); do + EXPECTED="$(psql_count "SELECT count(*) FROM expected_machines;")" + (( EXPECTED > 0 )) && break + sleep 5 +done +if (( EXPECTED > 0 )); then + ok "expected_machines=${EXPECTED} (registerExpectedMachines worked — RBAC grant present)" +else + warn "no expected_machines — this nico-api build lacks the Machineatron" + warn " AddExpectedMachine RBAC grant (403). Falling back to direct DB registration." + # scope strictly to BMC interfaces in the OOB prefix — with an unparsed + # prefix the filter would match admin-segment interfaces too and register + # them with the wrong factory password. + [[ "${OOB_PREFIX:-}" == */* ]] || die "cannot scope the expected_machines fallback: OOB prefix unknown" + psql_q "INSERT INTO expected_machines (id, serial_number, bmc_mac_address, bmc_username, bmc_password) + SELECT gen_random_uuid(), 'MAT-' || replace(mi.mac_address::text, ':', ''), mi.mac_address, 'root', '${FACTORY_HOST_BMC_PASSWORD}' + FROM machine_interfaces mi + JOIN machine_interface_addresses mia ON mia.interface_id = mi.id + WHERE mia.address << '${OOB_PREFIX}'::inet + AND NOT EXISTS (SELECT 1 FROM expected_machines em WHERE em.bmc_mac_address = mi.mac_address);" >/dev/null \ + || die "expected_machines DB fallback INSERT failed" + EXPECTED="$(psql_count "SELECT count(*) FROM expected_machines;")" + (( EXPECTED > 0 )) || die "expected_machines still 0 after fallback — machine creation cannot proceed" + ok "expected_machines=${EXPECTED} registered via DB fallback" +fi + +# --- kick exploration: clear any AvoidLockout latched before creds existed --- +# Exploration cycles that ran before Phase 4/5 completed record Unauthorized, +# which latches a self-perpetuating AvoidLockout in the exploration report. +# This mirrors the API's clear_last_known_error + request_exploration pair +# (crates/api-db/src/explored_endpoints.rs) — including the +# waiting_for_explorer_refresh flag, which gates preingestion until a fresh +# probe lands (skipping it would let preingestion act on the stale report). +psql_q "UPDATE explored_endpoints + SET exploration_report = jsonb_set(exploration_report, '{LastExplorationError}', 'null'::jsonb), + exploration_requested = true, + waiting_for_explorer_refresh = true;" >/dev/null || true +ok "cleared exploration lockouts + requested re-exploration" + +# machine target = one row per host + per DPU; wait scales with host count +MACHINE_TARGET=$(( HOST_COUNT * (1 + DPU_PER_HOST) )) +MACHINE_WAIT=$(( 420 + HOST_COUNT * 3 )); (( MACHINE_WAIT > 5400 )) && MACHINE_WAIT=5400 +info "waiting for explore → rotate → preingest → identify → create (target ${MACHINE_TARGET} machines, up to ${MACHINE_WAIT}s)..." +_end=$((SECONDS+MACHINE_WAIT)); MACHINES=0 +while (( SECONDS < _end )); do + MACHINES="$(psql_count "SELECT count(*) FROM machines;")" + (( MACHINES >= MACHINE_TARGET )) && break + # single round-trip for the progress line + _prog="$(psql_q "SELECT + (SELECT count(*) FILTER (WHERE exploration_report->'LastExplorationError' = 'null'::jsonb) FROM explored_endpoints) || '/' || + (SELECT count(*) FROM explored_managed_hosts);" || echo '?/?')" + info " endpoints_ok=${_prog%%/*}/${IFACES} managed_hosts=${_prog##*/} machines=${MACHINES} ..." + # Re-clear any AvoidLockout that latched during the wait (e.g. an + # exploration racing a mock reboot from preingestion's initial BMC reset). + # Idempotent; scoped to latched endpoints only so successful reports keep + # their state. + psql_q "UPDATE explored_endpoints + SET exploration_report = jsonb_set(exploration_report, '{LastExplorationError}', 'null'::jsonb), + exploration_requested = true, + waiting_for_explorer_refresh = true + WHERE exploration_report->'LastExplorationError'->>'Type' IN ('AvoidLockout','Unauthorized');" >/dev/null || true + # ...and UNPARK endpoints that have since explored clean: a lingering + # waiting_for_explorer_refresh gates them out of preingestion + # (find_preingest_not_waiting) even after a healthy report lands, stalling + # the pipeline at 'initial' indefinitely. + psql_q "UPDATE explored_endpoints + SET waiting_for_explorer_refresh = false, exploration_requested = false + WHERE waiting_for_explorer_refresh + AND exploration_report->'LastExplorationError' = 'null'::jsonb;" >/dev/null || true + sleep 25 +done +ENDPOINTS="$(psql_count "SELECT count(*) FROM explored_endpoints;")" +MHOSTS="$(psql_count "SELECT count(*) FROM explored_managed_hosts;")" +echo +if (( MACHINES >= MACHINE_TARGET )); then + ok "${GREEN}END TO END OK${NC} — endpoints=${ENDPOINTS}, managed_hosts=${MHOSTS}, machines=${MACHINES}/${MACHINE_TARGET}" +elif (( MACHINES > 0 )); then + ok "${GREEN}MACHINES CREATED${NC} (partial) — ${MACHINES}/${MACHINE_TARGET}; ingestion continuing in the background" +else + warn "machines not created yet (endpoints=${ENDPOINTS}, managed_hosts=${MHOSTS})" + warn " check: kubectl logs -n ${NICO_SYSTEM_NS} deploy/nico-api | grep -i 'site.explor\\|MissingCred\\|Refusing\\|Failed to create'" + warn " check: kubectl logs -n ${MAT_NAMESPACE} deploy/${RELEASE} | grep -iE 'No IP addresses|error'" + warn " a common cause: admin/OOB pool exhaustion — see the sizing output of Phase 6" +fi + +phase "Done" +info "machine-a-tron release ${RELEASE} deployed to ${MAT_NAMESPACE}." +info "Redeploy/iterate: re-run this script (idempotent) after 'export KUBECONFIG=...'." diff --git a/helm-prereqs/values/machine-a-tron-scale.yaml b/helm-prereqs/values/machine-a-tron-scale.yaml new file mode 100644 index 0000000000..856097927c --- /dev/null +++ b/helm-prereqs/values/machine-a-tron-scale.yaml @@ -0,0 +1,98 @@ +# ============================================================================= +# machine-a-tron-scale.yaml — LARGE-SCALE site values (MetalLB/nginx BMC mode) +# +# For scale testing NICo ingestion with hundreds to thousands of simulated +# hosts. Used by: MAT_MODE=scale setup-machine-a-tron.sh +# See docs/development/machine-a-tron-deployment.md and the chart README's +# "METALLB MODE" section. +# +# How this differs from Override Mode (values/machine-a-tron.yaml): +# - nginxBmcProxy.enabled: true — every simulated BMC gets its own +# LoadBalancer IP (one Service per BMC, chart cap 16384). nginx routes to +# the right mock via "Forwarded: host=". +# - machineATron.useSingleBmcMock: false — the mock registers one router per +# BMC keyed by its DHCP'd IP instead of one combined server. +# - site_explorer.bmc_proxy MUST BE UNSET in nico-core — site-explorer dials +# each BMC's dedicated IP directly (setup script removes it in scale mode). +# - Requires the simulated-oob / simulated-admin networks in the nico-core +# site config (added by the setup script's scale mode): +# [networks.simulated-oob] prefix 10.100.0.0/17 gateway 10.100.0.1 +# [networks.simulated-admin] prefix 10.102.0.0/18 gateway 10.102.0.1 +# The MetalLB ipRange below MUST sit inside the simulated-oob prefix so +# NICo-allocated BMC IPs match the MetalLB Service IPs (allocation is +# sequential on both sides). +# ============================================================================= + +## Registry-agnostic (like the NICo REST values): setup-machine-a-tron.sh +## injects image.repository = ${NICO_IMAGE_REGISTRY}/machine-a-tron and +## image.tag = ${MAT_IMAGE_TAG} at install time. Set them here only for +## direct helm installs without the script. +image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + +global: + imagePullSecrets: + - name: machine-a-tron-pull + +# Same SPIFFE identity requirement as Override Mode (see machine-a-tron.yaml). +certificate: + uris: + - "spiffe://nico.local/nico-system/sa/machine-a-tron" + +machineATron: + nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" + # Scale mode: no bmc_proxy anywhere — leave unset. + configureBmcProxyHost: "" + registerExpectedMachines: true + # true = SHARED REGISTRY mode (crates/machine-a-tron/src/main.rs): one + # server on bmc_mock_port routes to per-BMC mock routers via the + # "Forwarded: host=" header nginx injects. This is REQUIRED in K8s. + # false would make each mock bind its real BMC IP on the pod netns (bare- + # metal style) — nothing listens on 1266 and the pod crash-loops on probes. + useSingleBmcMock: true + +# Larger resources: one tokio task per host + per DPU; at thousands of +# machines the FSM/API chatter is CPU-bound. +resources: + limits: + cpu: "8" + memory: 8Gi + requests: + cpu: "2" + memory: 2Gi + +pods: + default: + machines: + # disable the chart-default example group (deep-merge would keep it) + rack-machines: null + dell-hosts: + hwType: dell_poweredge_r750 + # STAGE 1 default: 100 hosts × 2 DPUs = 300 BMC endpoints / LB Services. + # Raise per stage (1000 → 4500). Chart caps total BMCs at 16384. + # Override per run: HOST_COUNT / DPU_PER_HOST env to the setup script. + hostCount: 100 + dpuPerHostCount: 2 + vpcCount: 0 + subnetsPerVpc: 0 + dpuRebootDelay: 1 + hostRebootDelay: 1 + # Slower cadences than the dev profile — at thousands of machines the + # aggregate API/CPU load scales linearly with these. + scoutRunInterval: "300s" + runIntervalWorking: "5s" + runIntervalIdle: "60s" + networkStatusRunInterval: "120s" + templateDir: "/opt/machine-a-tron/templates" + # Gateways of the SIMULATED networks (added to nico-core by scale mode). + oobDhcpRelayAddress: "10.100.0.1" + adminDhcpRelayAddress: "10.102.0.1" + networkVirtualizationType: "" + dpusInNicMode: false + +extraEnv: + - name: RUST_LOG + value: "info" + diff --git a/helm-prereqs/values/machine-a-tron.yaml b/helm-prereqs/values/machine-a-tron.yaml new file mode 100644 index 0000000000..d25d4c385b --- /dev/null +++ b/helm-prereqs/values/machine-a-tron.yaml @@ -0,0 +1,105 @@ +# ============================================================================= +# machine-a-tron.yaml — site values template +# +# Copy this file and fill in site-specific values before deploying, OR let +# setup-machine-a-tron.sh consume it directly (it fills the site-specific +# fields from the running nico-core site config). +# See docs/development/machine-a-tron-deployment.md for the full guide. +# +# Deployment mode: Override Mode (nginxBmcProxy.enabled: false) +# site-explorer's Redfish calls are redirected to machine-a-tron's mock BMC. +# Not compatible with real hardware — use only on simulation-only clusters. +# +# IMPORTANT — the redirect is driven by nico-core, not this chart. nico-core's +# site_explorer config must contain (note the CROSS-NAMESPACE FQDN): +# [site_explorer] +# bmc_proxy = "nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local:1266" +# (Field is `bmc_proxy`, a single "host:port" string. The older +# override_target_ip/override_target_port fields are DEPRECATED, and +# override_target_host was never a valid field. setup-machine-a-tron.sh sets +# bmc_proxy for you.) The FQDN is REQUIRED: site-explorer runs in nico-system +# and cannot resolve the bare service name (which resolves against its own +# namespace) — a bare name yields a Redfish "connection refused". Setting +# bmc_proxy at launch also makes allow_changing_bmc_proxy default to true, +# which is what lets configureBmcProxyHost (below) work at runtime. +# ============================================================================= + +## Registry-agnostic (like the NICo REST values): setup-machine-a-tron.sh +## injects image.repository = ${NICO_IMAGE_REGISTRY}/machine-a-tron and +## image.tag = ${MAT_IMAGE_TAG} at install time. Set them here only for +## direct helm installs without the script. +image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + +global: + imagePullSecrets: + - name: machine-a-tron-pull + +# Override the SPIFFE URI so nico-api can authorize machine-a-tron calls. +# nico-api recognizes paths under /nico-system/sa/ and the RBAC rule for +# Machineatron expects SpiffeServiceIdentifier("machine-a-tron"). Without +# this override the cert-manager-issued cert uses the deployment namespace +# (/nico-mat/sa/nico-machine-a-tron) which is not in nico-api's trusted +# service_base_paths list and results in a 403 on every API call. +certificate: + uris: + - "spiffe://nico.local/nico-system/sa/machine-a-tron" + +machineATron: + nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" + # When set, machine-a-tron calls nico-api set_dynamic_config to point + # site_explorer.bmc_proxy at ":" at runtime. + # This ONLY succeeds if nico-core has allow_changing_bmc_proxy=true (which is + # the default once nico-core launches with bmc_proxy set). It is a + # belt-and-suspenders companion to the nico-core bmc_proxy setting — harmless + # when both are set to the same target. + # MUST be the cross-namespace FQDN (site-explorer is in nico-system and cannot + # resolve the bare name). Update the ".nico-mat." segment if you deploy to a + # different namespace. + configureBmcProxyHost: "nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local" + # Auto-register each mock host as an expected_machine (matched by BMC MAC). + # REQUIRED for end-to-end: site-explorer's MachineCreator refuses to create a + # managed host when no expected_machines row matches the discovered BMC MAC + # ("Refusing to create managed host, expected machines entry not found"). + registerExpectedMachines: true + +pods: + default: + machines: + # disable the chart-default example group (deep-merge would keep it) + rack-machines: null + dell-hosts: + hwType: dell_poweredge_r750 + # SIZING: machine-a-tron allocates one OOB DHCP IP per BMC interface — + # 1 per host + 1 per DPU. Total = hostCount + hostCount*dpuPerHostCount. + # This MUST fit the usable addresses in the OOB/underlay DHCP pool of your + # nico-core site config, or discovery reports "No IP addresses left in + # prefix ..." and machines never register. + # e.g. a /28 pool with a gateway and a few reserved addresses yields + # ~10 usable → hostCount:3 * dpuPerHostCount:2 = 9 fits; 5*2=15 does not. + # setup-machine-a-tron.sh estimates the pool size and warns if you overflow. + hostCount: 3 + dpuPerHostCount: 2 + vpcCount: 0 + subnetsPerVpc: 0 + dpuRebootDelay: 1 + hostRebootDelay: 1 + scoutRunInterval: "60s" + runIntervalWorking: "1s" + runIntervalIdle: "10s" + networkStatusRunInterval: "20s" + templateDir: "/opt/machine-a-tron/templates" + # Set to the gateway of the OOB/underlay network in nico-core site config. + # (setup-machine-a-tron.sh fills this from the running nico-core config.) + oobDhcpRelayAddress: "FILL_IN" + # Set to the gateway of the admin network in nico-core site config. + adminDhcpRelayAddress: "FILL_IN" + networkVirtualizationType: "" + dpusInNicMode: false + +extraEnv: + - name: RUST_LOG + value: "info" + diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 12a69869c4..e5e07b5738 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -44,3 +44,6 @@ dependencies: - name: unbound version: "0.1.0" condition: unbound.enabled + - name: nico-machine-a-tron + version: "0.1.0" + condition: nico-machine-a-tron.enabled diff --git a/helm/charts/nico-machine-a-tron/Chart.yaml b/helm/charts/nico-machine-a-tron/Chart.yaml new file mode 100644 index 0000000000..5f6ceac8d6 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: nico-machine-a-tron +description: Helm chart for Machine-A-Tron - a mock machine simulator for NICo development +type: application +version: 0.1.0 +appVersion: "latest" +keywords: + - nico + - machine-a-tron + - mock + - testing + - development diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md new file mode 100644 index 0000000000..cd9397f2e0 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/README.md @@ -0,0 +1,314 @@ +# Machine-A-Tron Helm Chart + +Helm chart for deploying Machine-A-Tron - a mock machine simulator for NICo testing. + +## Overview + +Machine-A-Tron creates simulated bare-metal machines that behave like real hosts, allowing you to: +- Test NICo without physical hardware +- Simulate multiple hosts, DPUs, switches and power shelves +- Perform load testing at scale (multiple pods, thousands of BMCs) +- Run simulations alongside real hardware + +## Deployment Modes + +| Mode | Use Case | Real HW Compatible | Network Setup | +|------|----------|-------------------|---------------| +| **Override Mode** | Development | No | Simple - single endpoint | +| **ClusterIP Mode** | Scale testing | Yes | Per-BMC ClusterIP services | + +--- + +## Mode 1: Override Mode (Development) + +**Use for development environments where only simulated machines are needed.** + +NICo's Site-Explorer is configured to redirect ALL Redfish calls to machine-a-tron. +Simple but **incompatible with real hardware**. + +### Setup + +```bash +helm upgrade --install nico ./helm \ + --namespace nico-mat \ + --set nico-machine-a-tron.enabled=true \ + --set nico-machine-a-tron.pods.default.machines.rack-machines.hostCount=10 \ + --set nico-machine-a-tron.pods.default.machines.rack-machines.dpuPerHostCount=2 +``` + +**NICo Site Config:** +```toml +[site_explorer] +override_target_host = "nico-machine-a-tron-bmc-mock" +override_target_port = 1266 +``` + +--- + +## Mode 2: ClusterIP Mode (Scale Testing) + +**Use for load testing environments where simulated machines run alongside real hardware.** + +Each simulated BMC gets a dedicated ClusterIP service. Supports multi-pod deployments. + +### Architecture + +```mermaid +flowchart TB + subgraph CIDR["ServiceCIDR: 10.100.0.0/20 (reserved)"] + direction TB + subgraph pod0["pod-0 (~14 racks)"] + cidr0["10.100.0.0/22
1021 IPs"] + end + subgraph pod1["pod-1 (~14 racks)"] + cidr1["10.100.4.0/22
1021 IPs"] + end + subgraph pod2["pod-2 (~14 racks)"] + cidr2["10.100.8.0/22
1021 IPs"] + end + end + + SE[Site-Explorer] --> cidr0 + SE --> cidr1 + SE --> cidr2 +``` + +### Prerequisites + +- **Kubernetes 1.29+** (ServiceCIDR API support) +- IP range within cluster's default service CIDR (e.g., `10.96.0.0/12`) + +### Single Pod Setup + +```yaml +# values.yaml +bmcServices: + enabled: true + serviceCIDR: + create: true + +pods: + default: + cidr: "10.100.0.0/22" # 1021 IPs for ~14 racks + machines: + compute: + hwType: wiwynn_gb200_nvl + hostCount: 252 # 18 trays × 14 racks + dpuPerHostCount: 2 # 2 BF3 per tray → 504 DPU BMCs + oobDhcpRelayAddress: "10.100.0.1" + switches: + hwType: nvidia_switch_nd5200_ld + hostCount: 126 # 9 switches × 14 racks + dpuPerHostCount: 0 + oobDhcpRelayAddress: "10.100.0.1" + power: + hwType: liteon_power_shelf + hostCount: 112 # 8 shelves × 14 racks + dpuPerHostCount: 0 + oobDhcpRelayAddress: "10.100.0.1" +``` + +**Total BMCs:** 252 + 504 + 126 + 112 = **994 BMCs** (fits in /22) + +### Multi-Pod Setup (Large Scale) + +For deployments exceeding 1021 BMCs, use multiple pods with separate CIDRs: + +```yaml +# values.yaml +bmcServices: + enabled: true + serviceCIDR: + create: true + cidr: "10.100.0.0/20" # Covers all pods + +pods: + pod-0: + cidr: "10.100.0.0/22" + machines: + compute: + hwType: wiwynn_gb200_nvl + hostCount: 252 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "10.100.0.1" + switches: + hwType: nvidia_switch_nd5200_ld + hostCount: 126 + oobDhcpRelayAddress: "10.100.0.1" + power: + hwType: liteon_power_shelf + hostCount: 112 + oobDhcpRelayAddress: "10.100.0.1" + + pod-1: + cidr: "10.100.4.0/22" + machines: + compute: + hwType: wiwynn_gb200_nvl + hostCount: 252 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "10.100.4.1" + switches: + hwType: nvidia_switch_nd5200_ld + hostCount: 126 + oobDhcpRelayAddress: "10.100.4.1" + power: + hwType: liteon_power_shelf + hostCount: 112 + oobDhcpRelayAddress: "10.100.4.1" + + pod-2: + cidr: "10.100.8.0/22" + machines: + # ... same pattern +``` + +### What Gets Created + +**Per pod:** +| Resource | Name Pattern | +|----------|--------------| +| Deployment | `nico-machine-a-tron-pod-0` | +| ConfigMap | `nico-machine-a-tron-pod-0-config-files` | +| Certificate | `nico-machine-a-tron-pod-0-certificate` | +| Service | `nico-machine-a-tron-pod-0-bmc-mock` | + +**Per BMC:** +| Resource | Name Pattern | +|----------|--------------| +| ClusterIP Service | `nico-machine-a-tron-bmc-10-100-0-2` | + +**Cluster-scoped:** +| Resource | Name | +|----------|------| +| ServiceCIDR | `nico-machine-a-tron-bmc-cidr` | + +### Scale Guidelines + +**BMC count per GB200 NVL72 rack:** +| Component | Count | BMCs per Unit | Total | +|-----------|-------|---------------|-------| +| Compute trays | 18 | 3 (1 tray + 2 BF3) | 54 | +| NVLink switches | 9 | 1 | 9 | +| Power shelves | 8 | 1 | 8 | +| **Total per rack** | | | **71** | + +**CIDR sizing:** +| CIDR | Usable IPs | Racks per Pod | +|------|------------|---------------| +| /24 | 253 | ~3 | +| /23 | 509 | ~7 | +| /22 | 1021 | ~14 | +| /21 | 2045 | ~28 | + +### NICo Configuration + +**DO NOT set `override_target_host`** - let NICo connect to actual BMC IPs: + +```toml +[site_explorer] +enabled = true +create_machines = true +# override_target_host = ... # DO NOT SET +``` + +**Network config per pod:** +```toml +[networks.pod-0-oob] +type = "underlay" +prefix = "10.100.0.0/22" +gateway = "10.100.0.1" + +[networks.pod-1-oob] +type = "underlay" +prefix = "10.100.4.0/22" +gateway = "10.100.4.1" +``` + +--- + +## Configuration Reference + +### Pod Configuration + +```yaml +pods: + : + cidr: "" # Required for bmcServices mode + machines: + : + hwType: wiwynn_gb200_nvl + hostCount: 10 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "10.100.0.1" + adminDhcpRelayAddress: "192.168.176.1" + # ... other machine settings +``` + +### BMC Services Configuration + +```yaml +bmcServices: + enabled: false + servicePort: 443 # External HTTPS port + serviceCIDR: + create: true + name: "" # Defaults to -bmc-cidr + cidr: "" # Auto-detected for single pod, required for multi-pod +``` + +### Supported Hardware Types + +| Type | Description | +|------|-------------| +| `supermicro_gb300_nvl` | Supermicro GB300 NVL | +| `nvidia_dgx_gb300` | NVIDIA DGX GB300 | +| `nvidia_dgx_h100` | NVIDIA DGX H100 | +| `wiwynn_gb200_nvl` | Wiwynn GB200 NVL | +| `lenovo_gb300_nvl` | Lenovo GB300 NVL | +| `dell_poweredge_r750` | Dell PowerEdge R750 | +| `liteon_power_shelf` | Liteon Power Shelf | +| `nvidia_switch_nd5200_ld` | NVIDIA ND5200 Switch | +| `generic_ami` | Generic AMI BMC | +| `generic_supermicro` | Generic Supermicro BMC | + +--- + +## Troubleshooting + +### ServiceCIDR Not Ready + +```bash +kubectl get servicecidr -o wide +``` + +Causes: +- CIDR outside cluster's default service CIDR +- Kubernetes version < 1.29 + +### Service IP Allocation Failed + +```bash +kubectl -n nico-mat describe svc nico-machine-a-tron-bmc-10-100-0-2 +``` + +Causes: +- IP already in use +- ServiceCIDR not ready + +### Pod Not Receiving Traffic + +Verify selector labels match: +```bash +# Check service selector +kubectl -n nico-mat get svc nico-machine-a-tron-bmc-10-100-0-2 -o jsonpath='{.spec.selector}' + +# Check pod labels +kubectl -n nico-mat get pods -l nvidia-infra-controller/pod-name=pod-0 --show-labels +``` + +### View Generated Config + +```bash +kubectl -n nico-mat get cm nico-machine-a-tron-pod-0-config-files -o yaml +``` diff --git a/helm/charts/nico-machine-a-tron/templates/_helpers.tpl b/helm/charts/nico-machine-a-tron/templates/_helpers.tpl new file mode 100644 index 0000000000..b27a1fdf47 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/_helpers.tpl @@ -0,0 +1,99 @@ +{{/* +Allow the release namespace to be overridden for multi-namespace deployments. +*/}} +{{- define "nico-machine-a-tron.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Resource name prefix. Defaults to the chart name; override with nameOverride. +*/}} +{{- define "nico-machine-a-tron.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "nico-machine-a-tron.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +What image to use: Use subchart-local image if defined, fall back on global +image. In devspace deployments, {{ include "nico-machine-a-tron.name" . }} gets its own image. +In other deployments, the main nico image contains all binaries. +*/}} +{{- define "nico-machine-a-tron.image" -}} +{{- if not (eq (toString (.Values.image.repository | default "")) "") }} +{{- .Values.image.repository }}:{{ .Values.image.tag | default "latest" }} +{{- else if and .Values.global.image (not (eq (toString (.Values.global.image.repository | default "")) "")) (not (eq (toString (.Values.global.image.tag | default "")) "")) }} +{{- .Values.global.image.repository }}:{{ .Values.global.image.tag }} +{{- else }} +{{- "nico:latest" }} +{{- end }} +{{- end }} + +{{- define "nico-machine-a-tron.labels" -}} +helm.sh/chart: {{ include "nico-machine-a-tron.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: site-controller +app.kubernetes.io/name: {{ include "nico-machine-a-tron.name" . }} +app.kubernetes.io/component: machine-a-tron +{{- end }} + +{{- define "nico-machine-a-tron.selectorLabels" -}} +app.kubernetes.io/name: {{ include "nico-machine-a-tron.name" . }} +app.kubernetes.io/component: machine-a-tron +{{- end }} + +{{- define "nico-machine-a-tron.certificateSpec" -}} +duration: {{ .global.certificate.duration }} +renewBefore: {{ .global.certificate.renewBefore }} +commonName: {{ printf "%s.%s.svc.cluster.local" (.cert.serviceName | default .svcName) (.cert.identityNamespace | default .namespace) }} +dnsNames: +{{- if .cert.dnsNames }} +{{- range .cert.dnsNames }} + - {{ . }} +{{- end }} +{{- else }} + - {{ printf "%s.%s.svc.cluster.local" (.cert.serviceName | default .svcName) (.cert.identityNamespace | default .namespace) }} +{{- if ne (toString .cert.includeShortDnsName) "false" }} + - {{ printf "%s.%s" (.cert.serviceName | default .svcName) (.cert.identityNamespace | default .namespace) }} +{{- end }} +{{- range .cert.extraDnsNames | default list }} + - {{ . }} +{{- end }} +{{- end }} +uris: +{{- if .cert.uris }} +{{- range .cert.uris }} + - {{ . }} +{{- end }} +{{- else }} + - {{ printf "spiffe://%s/%s/sa/%s" .global.spiffe.trustDomain (.cert.identityNamespace | default .namespace) (.cert.spiffeServiceName | default .cert.serviceName | default .svcName) }} +{{- range .cert.extraUris | default list }} + - {{ . }} +{{- end }} +{{- end }} +privateKey: + algorithm: {{ .global.certificate.privateKey.algorithm }} + size: {{ .global.certificate.privateKey.size }} +issuerRef: + kind: {{ .global.certificate.issuerRef.kind }} + name: {{ .global.certificate.issuerRef.name }} + group: {{ .global.certificate.issuerRef.group }} +secretName: {{ .name }} +{{- end }} + +{{- define "nico-machine-a-tron.serviceMonitorSpec" -}} +endpoints: + - honorLabels: false + interval: {{ .monitor.interval }} + port: {{ .port }} + scheme: http + scrapeTimeout: {{ .monitor.scrapeTimeout }} +namespaceSelector: + matchNames: + - {{ .namespace }} +selector: + matchLabels: + app.kubernetes.io/metrics: {{ .name }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-service-cidr.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-service-cidr.yaml new file mode 100644 index 0000000000..ea24422efd --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/bmc-service-cidr.yaml @@ -0,0 +1,55 @@ +{{- if .Values.bmcServices.enabled }} +{{- if .Values.bmcServices.serviceCIDR.create }} +{{- $root := . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Determine the ServiceCIDR to use: +1. If explicitly set in bmcServices.serviceCIDR.cidr, use that +2. Otherwise, use the CIDR from the first pod (single-pod case) +3. For multi-pod with different CIDRs, user must set serviceCIDR.cidr explicitly +*/}} +{{- $serviceCIDR := "" }} +{{- if .Values.bmcServices.serviceCIDR.cidr }} +{{- $serviceCIDR = .Values.bmcServices.serviceCIDR.cidr }} +{{- else }} +{{/* Collect all pod CIDRs */}} +{{- $cidrs := list }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $podConfig.cidr }} +{{- $cidrs = append $cidrs $podConfig.cidr }} +{{- end }} +{{- end }} +{{- if eq (len $cidrs) 1 }} +{{- $serviceCIDR = index $cidrs 0 }} +{{- else if gt (len $cidrs) 1 }} +{{- fail "Multiple pods with different CIDRs detected. Set bmcServices.serviceCIDR.cidr to a CIDR that covers all pod CIDRs." }} +{{- end }} +{{- end }} + +{{- if $serviceCIDR }} +--- +## ServiceCIDR reserves a dedicated IP range for BMC services. +## This prevents Kubernetes from allocating these IPs to other services. +## Requires Kubernetes 1.29+ (beta in 1.31+). +## +## The ServiceCIDR is cluster-scoped and persists independently of this release. +## Deleting the helm release will NOT delete the ServiceCIDR to prevent +## disruption to other releases using the same CIDR. +apiVersion: networking.k8s.io/v1beta1 +kind: ServiceCIDR +metadata: + name: {{ .Values.bmcServices.serviceCIDR.name | default (printf "%s-bmc-cidr" $baseName) }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + app.kubernetes.io/component: bmc-service-cidr + annotations: + helm.sh/resource-policy: keep + nvidia-infra-controller/purpose: "Reserved IP range for machine-a-tron BMC services" + nvidia-infra-controller/cidr: "{{ $serviceCIDR }}" +spec: + cidrs: + - {{ $serviceCIDR }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml new file mode 100644 index 0000000000..54f2353f2a --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml @@ -0,0 +1,131 @@ +{{- if .Values.bmcServices.enabled }} +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} +{{- $servicePort := .Values.bmcServices.servicePort | default 443 }} +{{- $targetPort := .Values.service.bmcMock.port | default 1266 }} + +{{/* +Count pods that have CIDR configured (for naming and selector logic) +*/}} +{{- $podsWithCIDR := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $podConfig.cidr }} +{{- $podsWithCIDR = add $podsWithCIDR 1 }} +{{- end }} +{{- end }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip pods without CIDR (e.g., the default pod when custom pods are defined) +*/}} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $podsWithCIDR 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} + +{{/* +Parse CIDR and calculate available IPs +*/}} +{{- $cidrParts := split "/" $podConfig.cidr }} +{{- $networkIP := $cidrParts._0 }} +{{- $prefixLen := $cidrParts._1 | int }} +{{- $ipParts := split "." $networkIP }} +{{- $octet1 := $ipParts._0 | int }} +{{- $octet2 := $ipParts._1 | int }} +{{- $octet3 := $ipParts._2 | int }} +{{- $octet4 := $ipParts._3 | int }} + +{{/* +Calculate total IPs and usable IPs: 2^(32-prefix) - 3 (network, gateway, broadcast) +*/}} +{{- $hostBits := sub 32 $prefixLen }} +{{- $totalIPs := 1 }} +{{- range $i := until (int $hostBits) }} +{{- $totalIPs = mul $totalIPs 2 }} +{{- end }} +{{- $usableIPs := sub $totalIPs 3 }} + +{{/* +Calculate total BMCs needed for this pod +*/}} +{{- $totalBMCs := 0 }} +{{- range $sectionName, $section := $podConfig.machines }} +{{- if $section }} +{{- $hostCount := $section.hostCount | default 1 }} +{{- $dpuCount := $section.dpuPerHostCount | default 0 }} +{{- $sectionBMCs := add $hostCount (mul $hostCount $dpuCount) }} +{{- $totalBMCs = add $totalBMCs $sectionBMCs }} +{{- end }} +{{- end }} + +{{/* +Validate we have enough IPs +*/}} +{{- if gt (int $totalBMCs) (int $usableIPs) }} +{{- fail (printf "INSUFFICIENT IP CAPACITY for pod '%s': %d BMCs requested but CIDR %s only has %d usable IPs" $podName $totalBMCs $podConfig.cidr $usableIPs) }} +{{- end }} + +{{/* +Generate ClusterIP services for each BMC +Services start at .2 (skipping .0 network and .1 gateway) +*/}} +{{- $startOctet4 := add $octet4 2 }} +{{- range $bmcIndex := until (int $totalBMCs) }} +{{- $offset := $bmcIndex }} +{{- $o4 := add $startOctet4 $offset }} +{{- $o3 := $octet3 }} +{{- $o2 := $octet2 }} +{{- $o1 := $octet1 }} + +{{/* Handle octet overflows */}} +{{- if gt $o4 255 }} +{{- $o3 = add $o3 (div $o4 256) }} +{{- $o4 = mod $o4 256 }} +{{- end }} +{{- if gt $o3 255 }} +{{- $o2 = add $o2 (div $o3 256) }} +{{- $o3 = mod $o3 256 }} +{{- end }} +{{- if gt $o2 255 }} +{{- $o1 = add $o1 (div $o2 256) }} +{{- $o2 = mod $o2 256 }} +{{- end }} + +{{- $currentIP := printf "%d.%d.%d.%d" (int $o1) (int $o2) (int $o3) (int $o4) }} +{{- $safeName := $currentIP | replace "." "-" }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $baseName }}-bmc-{{ $safeName }} + namespace: {{ $namespace }} + labels: + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + app.kubernetes.io/component: bmc-service + nvidia-infra-controller/bmc-ip: "{{ $currentIP }}" + nvidia-infra-controller/bmc-index: "{{ $bmcIndex }}" + nvidia-infra-controller/pod-name: {{ $podName | quote }} + annotations: + nvidia-infra-controller/purpose: "ClusterIP service for simulated BMC" + nvidia-infra-controller/pod-cidr: "{{ $podConfig.cidr }}" +spec: + type: ClusterIP + clusterIP: {{ $currentIP }} + selector: + {{- include "nico-machine-a-tron.selectorLabels" $root | nindent 4 }} + {{- if gt $podsWithCIDR 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} + ports: + - name: https + port: {{ $servicePort }} + targetPort: {{ $targetPort }} + protocol: TCP +{{- end }} +{{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/certificate.yaml b/helm/charts/nico-machine-a-tron/templates/certificate.yaml new file mode 100644 index 0000000000..17d9401239 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/certificate.yaml @@ -0,0 +1,55 @@ +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Count active pods (same logic as deployment.yaml) +*/}} +{{- $activePods := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $root.Values.bmcServices.enabled }} +{{- if $podConfig.cidr }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- else }} +{{- if $podConfig.machines }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- end }} +{{- end }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip conditions (same as deployment.yaml) +*/}} +{{- if $root.Values.bmcServices.enabled }} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} +{{- else }} +{{- if not $podConfig.machines }} +{{- continue }} +{{- end }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $activePods 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ $name }}-certificate + namespace: {{ $namespace }} + labels: + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} +spec: + {{- $bmcMockDns := printf "%s-bmc-mock.%s.svc.cluster.local" $name $namespace }} + {{- $extraDns := prepend ($root.Values.certificate.extraDnsNames | default list) $bmcMockDns }} + {{- $certWithExtra := merge (dict "extraDnsNames" $extraDns) $root.Values.certificate }} + {{- include "nico-machine-a-tron.certificateSpec" (dict "name" (printf "%s-certificate" $name) "svcName" $name "namespace" $namespace "cert" $certWithExtra "global" $root.Values.global) | nindent 2 }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/configmap.yaml b/helm/charts/nico-machine-a-tron/templates/configmap.yaml new file mode 100644 index 0000000000..79e4f48ae8 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/configmap.yaml @@ -0,0 +1,119 @@ +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Count active pods (same logic as deployment.yaml) +*/}} +{{- $activePods := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $root.Values.bmcServices.enabled }} +{{- if $podConfig.cidr }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- else }} +{{- if $podConfig.machines }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Machine defaults */}} +{{- $defaults := .Values.machineDefaults | default dict }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip conditions (same as deployment.yaml) +*/}} +{{- if $root.Values.bmcServices.enabled }} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} +{{- else }} +{{- if not $podConfig.machines }} +{{- continue }} +{{- end }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $activePods 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-config-files + namespace: {{ $namespace }} + labels: + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} +data: + mat.toml: | + {{- if and $root.Values.configFiles.matConfigs (index $root.Values.configFiles.matConfigs $podName) }} + {{- index $root.Values.configFiles.matConfigs $podName | nindent 4 }} + {{- else }} + # Machine-A-Tron configuration - generated by Helm + # Pod: {{ $podName }} + {{- if $podConfig.cidr }} + # CIDR: {{ $podConfig.cidr }} + {{- end }} + carbide_api_url = {{ $root.Values.machineATron.nicoApiUrl | quote }} + interface = {{ $root.Values.machineATron.interface | quote }} + tui_enabled = {{ $root.Values.machineATron.tuiEnabled }} + {{- if $root.Values.machineATron.logFile }} + log_file = {{ $root.Values.machineATron.logFile | quote }} + {{- end }} + bmc_mock_port = {{ $root.Values.service.bmcMock.port }} + use_single_bmc_mock = {{ $root.Values.machineATron.useSingleBmcMock }} + mock_bmc_ssh_server = {{ $root.Values.machineATron.mockBmcSshServer }} + {{- if $root.Values.machineATron.mockBmcSshPort }} + mock_bmc_ssh_port = {{ $root.Values.machineATron.mockBmcSshPort }} + {{- end }} + {{- if $root.Values.machineATron.configureBmcProxyHost }} + configure_carbide_bmc_proxy_host = {{ $root.Values.machineATron.configureBmcProxyHost | quote }} + {{- end }} + {{- if $root.Values.persistence.enabled }} + persist_dir = {{ $root.Values.machineATron.persistDir | quote }} + {{- end }} + cleanup_on_quit = {{ $root.Values.machineATron.cleanupOnQuit }} + register_expected_machines = {{ $root.Values.machineATron.registerExpectedMachines }} + {{- if $root.Values.machineATron.hostBmcPassword }} + host_bmc_password = {{ $root.Values.machineATron.hostBmcPassword | quote }} + {{- end }} + {{- if $root.Values.machineATron.dpuBmcPassword }} + dpu_bmc_password = {{ $root.Values.machineATron.dpuBmcPassword | quote }} + {{- end }} + api_refresh_interval = {{ $root.Values.machineATron.apiRefreshInterval | quote }} + + {{- range $sectionName, $section := $podConfig.machines }} + {{- if $section }} + + [machines.{{ $sectionName }}] + {{- if $section.hwType }} + hw_type = {{ $section.hwType | quote }} + {{- end }} + host_count = {{ $section.hostCount | default 1 }} + vpc_count = {{ $section.vpcCount | default $defaults.vpcCount | default 0 }} + subnets_per_vpc = {{ $section.subnetsPerVpc | default $defaults.subnetsPerVpc | default 0 }} + dpu_per_host_count = {{ $section.dpuPerHostCount | default 0 }} + dpu_reboot_delay = {{ $section.dpuRebootDelay | default $defaults.dpuRebootDelay | default 1 }} + host_reboot_delay = {{ $section.hostRebootDelay | default $defaults.hostRebootDelay | default 1 }} + scout_run_interval = {{ $section.scoutRunInterval | default $defaults.scoutRunInterval | default "60s" | quote }} + run_interval_working = {{ $section.runIntervalWorking | default $defaults.runIntervalWorking | default "1s" | quote }} + run_interval_idle = {{ $section.runIntervalIdle | default $defaults.runIntervalIdle | default "10s" | quote }} + network_status_run_interval = {{ $section.networkStatusRunInterval | default $defaults.networkStatusRunInterval | default "20s" | quote }} + template_dir = {{ $section.templateDir | default $defaults.templateDir | default "/opt/machine-a-tron/templates" | quote }} + oob_dhcp_relay_address = {{ $section.oobDhcpRelayAddress | default "192.168.192.1" | quote }} + admin_dhcp_relay_address = {{ $section.adminDhcpRelayAddress | default "192.168.176.1" | quote }} + {{- $nvType := $section.networkVirtualizationType | default $defaults.networkVirtualizationType }} + {{- if $nvType }} + network_virtualization_type = {{ $nvType | quote }} + {{- end }} + dpus_in_nic_mode = {{ $section.dpusInNicMode | default $defaults.dpusInNicMode | default false }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/deployment.yaml b/helm/charts/nico-machine-a-tron/templates/deployment.yaml new file mode 100644 index 0000000000..05074aa06f --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/deployment.yaml @@ -0,0 +1,178 @@ +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Count pods that should be deployed: +- When bmcServices.enabled: only pods with CIDR +- Otherwise: all pods with machines +*/}} +{{- $activePods := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $root.Values.bmcServices.enabled }} +{{- if $podConfig.cidr }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- else }} +{{- if $podConfig.machines }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- end }} +{{- end }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip conditions: +- When bmcServices.enabled: skip pods without CIDR +- Otherwise: skip pods without machines +*/}} +{{- if $root.Values.bmcServices.enabled }} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} +{{- else }} +{{- if not $podConfig.machines }} +{{- continue }} +{{- end }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $activePods 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $name }} + namespace: {{ $namespace }} + labels: + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} + annotations: + {{- tpl (toYaml $root.Values.annotations) $root | nindent 4 }} +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "nico-machine-a-tron.selectorLabels" $root | nindent 6 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} + template: + metadata: + labels: + {{- include "nico-machine-a-tron.labels" $root | nindent 8 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} + annotations: + kubectl.kubernetes.io/default-container: {{ $baseName }} + spec: + automountServiceAccountToken: {{ $root.Values.automountServiceAccountToken }} + serviceAccountName: {{ $baseName }} + terminationGracePeriodSeconds: {{ $root.Values.terminationGracePeriodSeconds }} + {{- with $root.Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ $baseName }} + image: "{{ include "nico-machine-a-tron.image" $root }}" + imagePullPolicy: {{ ($root.Values.global.image).pullPolicy | default $root.Values.image.pullPolicy | default "IfNotPresent" }} + command: + {{- toYaml $root.Values.command | nindent 12 }} + env: + - name: VAULT_ROLE_ID + valueFrom: + secretKeyRef: + name: {{ $root.Values.envFrom.vaultApproleTokens.secretName }} + key: VAULT_ROLE_ID + optional: true + - name: VAULT_SECRET_ID + valueFrom: + secretKeyRef: + name: {{ $root.Values.envFrom.vaultApproleTokens.secretName }} + key: VAULT_SECRET_ID + optional: true + - name: VAULT_TOKEN + valueFrom: + secretKeyRef: + name: {{ $root.Values.envFrom.vaultToken.secretName }} + key: token + optional: true + - name: VAULT_ADDR + valueFrom: + configMapKeyRef: + name: {{ $root.Values.envFrom.vaultClusterInfo.configMapName }} + key: VAULT_SERVICE + optional: true + - name: MACHINE_A_TRON_CONFIG_PATH + value: "/etc/nvidia-infra-controller/machine-a-tron/mat.toml" + - name: FORGE_ROOT_CA_PATH + value: "/var/run/secrets/spiffe.io/ca.crt" + - name: CLIENT_CERT_PATH + value: "/var/run/secrets/spiffe.io/tls.crt" + - name: CLIENT_KEY_PATH + value: "/var/run/secrets/spiffe.io/tls.key" + - name: RUST_BACKTRACE + value: {{ $root.Values.env.RUST_BACKTRACE | quote }} + - name: RUST_LIB_BACKTRACE + value: {{ $root.Values.env.RUST_LIB_BACKTRACE | quote }} + {{- with $root.Values.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: redfish + containerPort: {{ $root.Values.service.bmcMock.port }} + - name: ssh + containerPort: {{ $root.Values.machineATron.mockBmcSshPort }} + - name: metrics + containerPort: {{ $root.Values.service.metrics.port }} + livenessProbe: + {{- toYaml $root.Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml $root.Values.readinessProbe | nindent 12 }} + volumeMounts: + - name: config-files + mountPath: /etc/forge/machine-a-tron + - name: config-files + mountPath: /etc/nvidia-infra-controller/machine-a-tron + - name: spiffe + mountPath: "/var/run/secrets/spiffe.io" + readOnly: true + - name: nico-roots + mountPath: "/var/run/secrets/forge-roots" + readOnly: true + - name: nico-roots + mountPath: "/var/run/secrets/nico-roots" + readOnly: true + {{- if $root.Values.persistence.enabled }} + - name: data + mountPath: {{ $root.Values.machineATron.persistDir }} + {{- if gt $activePods 1 }} + subPath: {{ $podName }} + {{- end }} + {{- end }} + resources: + {{- toYaml $root.Values.resources | nindent 12 }} + volumes: + - name: config-files + configMap: + name: {{ $name }}-config-files + - name: spiffe + secret: + secretName: {{ $name }}-certificate + - name: nico-roots + secret: + secretName: nico-roots + {{- if $root.Values.persistence.enabled }} + - name: data + persistentVolumeClaim: + claimName: {{ $baseName }}-data + {{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/external-service.yaml b/helm/charts/nico-machine-a-tron/templates/external-service.yaml new file mode 100644 index 0000000000..33f262c97f --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/external-service.yaml @@ -0,0 +1,28 @@ +{{/* +External LoadBalancer service for Override Mode (development). +Only used when bmcServices is disabled - provides a single external endpoint. +When bmcServices.enabled, use ClusterIP services instead for per-BMC routing. +*/}} +{{- if and .Values.externalService.enabled (not .Values.bmcServices.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-external + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + {{- with .Values.externalService.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.externalService.type }} + externalTrafficPolicy: {{ .Values.externalService.externalTrafficPolicy }} + selector: + {{- include "nico-machine-a-tron.selectorLabels" . | nindent 4 }} + ports: + - port: {{ .Values.service.bmcMock.port }} + name: bmc-mock + targetPort: redfish + protocol: TCP +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/metrics-service.yaml b/helm/charts/nico-machine-a-tron/templates/metrics-service.yaml new file mode 100644 index 0000000000..f7dafc029a --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/metrics-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-metrics + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + app.kubernetes.io/metrics: {{ include "nico-machine-a-tron.name" . }} +spec: + selector: + {{- include "nico-machine-a-tron.selectorLabels" . | nindent 4 }} + ports: + - port: {{ .Values.service.metrics.port }} + name: metrics + protocol: TCP diff --git a/helm/charts/nico-machine-a-tron/templates/namespace.yaml b/helm/charts/nico-machine-a-tron/templates/namespace.yaml new file mode 100644 index 0000000000..bc0d92cdfd --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/namespace.yaml @@ -0,0 +1,10 @@ +{{- if and .Values.namespaceOverride .Values.createNamespace }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ include "nico-machine-a-tron.namespace" . }} + annotations: + helm.sh/resource-policy: keep + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/pvc.yaml b/helm/charts/nico-machine-a-tron/templates/pvc.yaml new file mode 100644 index 0000000000..7ac4d8de03 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/pvc.yaml @@ -0,0 +1,18 @@ +{{- if .Values.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-data + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +spec: + accessModes: + {{- toYaml .Values.persistence.accessModes | nindent 4 }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/rbac.yaml b/helm/charts/nico-machine-a-tron/templates/rbac.yaml new file mode 100644 index 0000000000..5cc13d0f4b --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/rbac.yaml @@ -0,0 +1,27 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +rules: + - apiGroups: ["cert-manager.io"] + resources: ["certificaterequests"] + verbs: ["create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "nico-machine-a-tron.name" . }} +subjects: + - kind: ServiceAccount + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} diff --git a/helm/charts/nico-machine-a-tron/templates/service-account.yaml b/helm/charts/nico-machine-a-tron/templates/service-account.yaml new file mode 100644 index 0000000000..d7a1fe8a92 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/service-account.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} diff --git a/helm/charts/nico-machine-a-tron/templates/service-monitor.yaml b/helm/charts/nico-machine-a-tron/templates/service-monitor.yaml new file mode 100644 index 0000000000..36c039fc0b --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/service-monitor.yaml @@ -0,0 +1,11 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +spec: + {{- include "nico-machine-a-tron.serviceMonitorSpec" (dict "name" (include "nico-machine-a-tron.name" .) "monitor" .Values.serviceMonitor "port" "metrics" "namespace" (include "nico-machine-a-tron.namespace" .)) | nindent 2 }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/service.yaml b/helm/charts/nico-machine-a-tron/templates/service.yaml new file mode 100644 index 0000000000..a8faab7fc6 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/service.yaml @@ -0,0 +1,68 @@ +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Count active pods (same logic as deployment.yaml) +*/}} +{{- $activePods := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $root.Values.bmcServices.enabled }} +{{- if $podConfig.cidr }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- else }} +{{- if $podConfig.machines }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- end }} +{{- end }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip conditions (same as deployment.yaml) +*/}} +{{- if $root.Values.bmcServices.enabled }} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} +{{- else }} +{{- if not $podConfig.machines }} +{{- continue }} +{{- end }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $activePods 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }}-bmc-mock + namespace: {{ $namespace }} + labels: + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + app: {{ $baseName }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} +spec: + selector: + {{- include "nico-machine-a-tron.selectorLabels" $root | nindent 4 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} + ports: + - port: {{ $root.Values.service.bmcMock.port }} + name: redfish + targetPort: redfish + protocol: TCP + {{- if $root.Values.machineATron.mockBmcSshServer }} + - port: {{ $root.Values.service.ssh.port }} + name: ssh + targetPort: ssh + protocol: TCP + {{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/tests/certificate_test.yaml b/helm/charts/nico-machine-a-tron/tests/certificate_test.yaml new file mode 100644 index 0000000000..20d7dd901c --- /dev/null +++ b/helm/charts/nico-machine-a-tron/tests/certificate_test.yaml @@ -0,0 +1,42 @@ +suite: certificate tests +templates: + - certificate.yaml +tests: + - it: should create a certificate with default values + asserts: + - isKind: + of: Certificate + - equal: + path: metadata.name + value: nico-machine-a-tron-certificate + + - it: should use custom certificate settings + set: + certificate: + serviceName: custom-service + extraDnsNames: + - extra.example.com + asserts: + - isKind: + of: Certificate + + - it: should create per-pod certificates in multi-pod mode + set: + bmcServices: + enabled: true + pods: + mat-0: + cidr: "10.100.0.0/22" + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 5 + mat-1: + cidr: "10.100.4.0/22" + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 5 + asserts: + - hasDocuments: + count: 2 diff --git a/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml b/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml new file mode 100644 index 0000000000..41f46ac80c --- /dev/null +++ b/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml @@ -0,0 +1,90 @@ +suite: configmap tests +templates: + - configmap.yaml +tests: + - it: should create configmap with default machine config + asserts: + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: nico-machine-a-tron-config-files + - matchRegex: + path: data["mat.toml"] + pattern: "host_count = 10" + - matchRegex: + path: data["mat.toml"] + pattern: "use_single_bmc_mock = true" + + - it: should configure custom host count via pods structure + set: + pods: + default: + machines: + dell-hosts: + hwType: dell_poweredge_r750 + hostCount: 20 + dpuPerHostCount: 2 + asserts: + - matchRegex: + path: data["mat.toml"] + pattern: "host_count = 20" + - matchRegex: + path: data["mat.toml"] + pattern: "dpu_per_host_count = 2" + - matchRegex: + path: data["mat.toml"] + pattern: 'hw_type = "dell_poweredge_r750"' + + - it: should allow full config override via matConfigs + set: + configFiles: + matConfigs: + default: | + carbide_api_url = "https://custom-api:443" + [machines.custom] + host_count = 5 + asserts: + - matchRegex: + path: data["mat.toml"] + pattern: "custom-api" + + - it: should create per-pod configmaps in multi-pod mode + set: + bmcServices: + enabled: true + pods: + mat-0: + cidr: "10.100.0.0/22" + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 10 + mat-1: + cidr: "10.100.4.0/22" + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 20 + asserts: + - hasDocuments: + count: 2 + + - it: should use machineDefaults for all machine groups + set: + machineDefaults: + runIntervalIdle: "30s" + scoutRunInterval: "120s" + pods: + default: + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 5 + asserts: + - matchRegex: + path: data["mat.toml"] + pattern: 'run_interval_idle = "30s"' + - matchRegex: + path: data["mat.toml"] + pattern: 'scout_run_interval = "120s"' diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml new file mode 100644 index 0000000000..5d3fa1f308 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -0,0 +1,307 @@ +## ============================================================================= +## nico-machine-a-tron - Mock machine simulator for NICo testing +## ============================================================================= +## +## DEPLOYMENT MODES: +## +## 1. OVERRIDE MODE (Development) - bmcServices.enabled: false +## - NICo Site-Explorer redirects ALL Redfish calls to machine-a-tron +## - Simple setup, single pod +## - INCOMPATIBLE with real hardware +## +## 2. CLUSTERIP MODE (Large-Scale Testing) - bmcServices.enabled: true +## - Each simulated BMC gets dedicated ClusterIP service +## - Supports multi-pod deployments via `pods` configuration +## - Compatible with real hardware +## +## See README.md for detailed setup instructions. +## +## ============================================================================= + +global: + certificate: + duration: 720h0m0s + renewBefore: 360h0m0s + privateKey: + algorithm: ECDSA + size: 384 + issuerRef: + kind: ClusterIssuer + name: vault-nico-issuer + group: cert-manager.io + spiffe: + trustDomain: nico.local + labels: + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: site-controller + +image: + ## Container image repository. When empty, uses global.image.repository or defaults to nico:latest + repository: "" + ## Container image tag. Defaults to chart appVersion + tag: "" + pullPolicy: IfNotPresent + +namespaceOverride: "" + +## Create namespace when namespaceOverride is set +createNamespace: true + +automountServiceAccountToken: true + +## Graceful shutdown timeout (seconds) +terminationGracePeriodSeconds: 60 + +annotations: + configmap.reloader.stakater.com/reload: '{{ include "nico-machine-a-tron.name" . }}-config-files' + +resources: + limits: + cpu: 4 + memory: 4Gi + requests: + cpu: 500m + memory: 1Gi + +service: + bmcMock: + port: 1266 + ssh: + port: 22 + targetPort: 2222 + metrics: + port: 9090 + +certificate: + serviceName: "" + spiffeServiceName: "" + identityNamespace: "" + dnsNames: [] + uris: [] + extraDnsNames: [] + extraUris: [] + +command: + - /bin/sh + - -c + - /opt/machine-a-tron/bin/machine-a-tron + +env: + RUST_BACKTRACE: "full" + RUST_LIB_BACKTRACE: "0" + +extraEnv: [] + +envFrom: + vaultApproleTokens: + secretName: nico-vault-approle-tokens + vaultToken: + secretName: nico-vault-token + vaultClusterInfo: + configMapName: vault-cluster-info + +livenessProbe: + tcpSocket: + port: redfish + initialDelaySeconds: 30 + periodSeconds: 30 + failureThreshold: 3 + successThreshold: 1 + timeoutSeconds: 10 + +readinessProbe: + tcpSocket: + port: redfish + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + timeoutSeconds: 5 + +serviceMonitor: + enabled: false + interval: 30s + scrapeTimeout: 25s + +## ============================================================================= +## Machine-a-tron Core Settings (shared across all pods) +## ============================================================================= +machineATron: + nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" + interface: "NOTUSED" + tuiEnabled: false + logFile: "" + useSingleBmcMock: true + mockBmcSshServer: true + mockBmcSshPort: 2222 + configureBmcProxyHost: "" + persistDir: "/tmp/machine-a-tron-data" + cleanupOnQuit: false + registerExpectedMachines: true + hostBmcPassword: "" + dpuBmcPassword: "" + apiRefreshInterval: "2s" + +## ============================================================================= +## Machine Defaults - Shared settings for all machine groups +## ============================================================================= +## These defaults apply to all machine groups in all pods. +## Override per-group by specifying the value in pods.*.machines.*. +## +machineDefaults: + vpcCount: 0 + subnetsPerVpc: 0 + dpuRebootDelay: 1 + hostRebootDelay: 1 + scoutRunInterval: "60s" + runIntervalWorking: "1s" + runIntervalIdle: "10s" + networkStatusRunInterval: "20s" + templateDir: "/opt/machine-a-tron/templates" + networkVirtualizationType: "" + dpusInNicMode: false + +## ============================================================================= +## Pod Configurations - One pod per config entry +## ============================================================================= +## Each entry creates: +## - 1 Deployment (pod) +## - 1 ConfigMap with mat.toml +## - 1 Certificate +## - 1 internal Service +## - N ClusterIP BMC services (when bmcServices.enabled) +## +## SCALE: Use /22 CIDR per pod = 1021 usable IPs = ~14 racks worth of BMCs +## +## BMC count per GB200 NVL72 rack: 71 +## - Compute trays: 18 × 3 = 54 +## - NVLink switches: 9 × 1 = 9 +## - Power shelves: 8 × 1 = 8 +## +## Per-machine group, only these fields are typically needed: +## - hwType: Hardware type (required) +## - hostCount: Number of hosts (required) +## - dpuPerHostCount: DPUs per host (default: 0) +## - oobDhcpRelayAddress: OOB network gateway +## - adminDhcpRelayAddress: Admin network gateway +## +## All other settings inherit from machineDefaults above. +## +pods: + ## Single-pod configuration is used when bmcServices.enabled=false (by default) + ## For multi-pod deployments, add more entries below and enable bmcServices + default: + cidr: "" + + ## Machine groups for this pod + machines: + rack-machines: + hwType: wiwynn_gb200_nvl + hostCount: 10 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "192.168.192.1" + adminDhcpRelayAddress: "192.168.176.1" + +## ============================================================================= +## Multi-Pod Example (bmcServices.enabled is required) +## ============================================================================= +## Uncomment and customize for large-scale deployments: +## +# mat-0: +# cidr: "10.100.0.0/22" # 1021 IPs for ~14 racks +# machines: +# compute: +# hwType: wiwynn_gb200_nvl +# hostCount: 252 # 18 trays × 14 racks +# dpuPerHostCount: 2 # 2 BF3 per tray +# oobDhcpRelayAddress: "10.100.0.1" +# adminDhcpRelayAddress: "192.168.176.1" +# switches: +# hwType: nvidia_switch_nd5200_ld +# hostCount: 126 # 9 switches × 14 racks +# oobDhcpRelayAddress: "10.100.0.1" +# adminDhcpRelayAddress: "192.168.176.1" +# power: +# hwType: liteon_power_shelf +# hostCount: 112 # 8 shelves × 14 racks +# oobDhcpRelayAddress: "10.100.0.1" +# adminDhcpRelayAddress: "192.168.176.1" +# +# mat-1: +# cidr: "10.100.4.0/22" # Next /22 block +# machines: +# compute: +# hwType: wiwynn_gb200_nvl +# hostCount: 252 +# dpuPerHostCount: 2 +# oobDhcpRelayAddress: "10.100.4.1" +# adminDhcpRelayAddress: "192.168.176.1" +# # ... same pattern + +## ============================================================================= +## BMC Services Configuration +## ============================================================================= +bmcServices: + enabled: false + + ## External port for BMC services (matches real BMC HTTPS port) + servicePort: 443 + + ## ServiceCIDR reserves the IP range (Kubernetes 1.29+) + serviceCIDR: + create: true + ## Name for the ServiceCIDR (cluster-scoped) + name: "" + ## Combined CIDR covering all pods (auto-calculated if empty for single pod) + ## Set manually if pods have non-contiguous CIDRs + cidr: "" + +## Persistence volume for machine state +persistence: + enabled: false + storageClass: "" + size: 1Gi + accessModes: + - ReadWriteOnce + +externalService: + enabled: false + type: LoadBalancer + externalTrafficPolicy: Local + annotations: {} + +configFiles: + ## Override mat.toml per pod (optional) + ## Key = pod name, value = full config content + ## When set, this completely replaces the auto-generated config for that pod. + ## + ## Example - simple single-pod config: + # matConfigs: + # default: | + # carbide_api_url = "https://nico-api.nico-system.svc.cluster.local:1079" + # interface = "NOTUSED" + # tui_enabled = false + # bmc_mock_port = 1266 + # use_single_bmc_mock = true + # mock_bmc_ssh_server = true + # mock_bmc_ssh_port = 2222 + # cleanup_on_quit = false + # register_expected_machines = true + # api_refresh_interval = "2s" + # + # [machines.compute] + # hw_type = "supermicro_gb300_nvl" + # host_count = 10 + # dpu_per_host_count = 2 + # oob_dhcp_relay_address = "10.100.0.1" + # admin_dhcp_relay_address = "192.168.176.1" + # template_dir = "/opt/machine-a-tron/templates" + # + # [machines.switches] + # hw_type = "nvidia_switch_nd5200_ld" + # host_count = 5 + # dpu_per_host_count = 0 + # oob_dhcp_relay_address = "10.100.0.1" + # admin_dhcp_relay_address = "192.168.176.1" + # template_dir = "/opt/machine-a-tron/templates" + matConfigs: {} diff --git a/helm/templates/NOTES.txt b/helm/templates/NOTES.txt index bfb2ae32c4..685e307a3b 100644 --- a/helm/templates/NOTES.txt +++ b/helm/templates/NOTES.txt @@ -30,6 +30,9 @@ Deployed sub-charts: {{- if index .Values "nico-ssh-console-rs" "enabled" | default true }} ✓ nico-ssh-console-rs {{- end }} +{{- if index .Values "nico-machine-a-tron" "enabled" | default false }} + ✓ nico-machine-a-tron (mock machines for dev/test) +{{- end }} {{- if index .Values "unbound" "enabled" | default false }} ✓ unbound {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index 92e7a81bdc..1ea49ea43f 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -165,3 +165,13 @@ nico-ssh-console-rs: ## --------------------------------------------------------------------------- unbound: enabled: false + +## --------------------------------------------------------------------------- +## nico-machine-a-tron - Mock machine simulator (development/testing only) +## Simulates bare-metal machines, DPUs, switches and power shelves for +## testing NICo without physical hardware. NOT for production use. +## Deploy standalone for dev environments: +## helm install mat ./helm/charts/nico-machine-a-tron -n nico-system ... +## --------------------------------------------------------------------------- +nico-machine-a-tron: + enabled: false