Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
86c3211
feat(helm): nico machine-a-tron
akorobkov-nvda Jun 10, 2026
d21d03b
fix(helm): nico machine-a-tron command
akorobkov-nvda Jun 10, 2026
052290b
fix(helm): nico machine-a-tron config
akorobkov-nvda Jun 10, 2026
025820c
fix(helm): nico machine-a-tron requirements
akorobkov-nvda Jun 10, 2026
af5a884
fix(helm): nico machine-a-tron machines config
akorobkov-nvda Jun 10, 2026
63f3e58
feat(helm): nico machine-a-tron graceful shutdown support
akorobkov-nvda Jun 22, 2026
a4979d8
feat(helm): nico machine-a-tron readme
akorobkov-nvda Jun 22, 2026
593860b
fix(helm): nico machine-a-tron umbrella chart metadata
akorobkov-nvda Jun 22, 2026
14a5fe2
fix(helm): nico machine-a-tron chart review
akorobkov-nvda Jun 22, 2026
74c04c8
feat(helm): nico machine-a-tron large scale
akorobkov-nvda Jun 23, 2026
6e9eea2
chore(helm): nico machine-a-tron nginx proxy rename
akorobkov-nvda Jun 23, 2026
0ccdee4
chore(helm): nico machine-a-tron readme
akorobkov-nvda Jun 23, 2026
b9b66f0
fix(helm): nico machine-a-tron values
akorobkov-nvda Jun 23, 2026
185d041
feat(helm): nico machine-a-tron migrate to clusterip
akorobkov-nvda Jul 7, 2026
ba12d66
test(helm): nico machine-a-tron
akorobkov-nvda Jul 7, 2026
4779564
feat(dev): devspace use machine-a-tron chart
akorobkov-nvda Jul 8, 2026
deac45b
feat(machine-a-tron): end-to-end deployment + scale testing to 4500 h…
shayan1995 Jul 8, 2026
cf497e4
docs(machine-a-tron): 4500-node quick start, measured timings, final …
shayan1995 Jul 9, 2026
6d7b87d
Merge pull request #2 from shayan1995/machine-a-tron-e2e-on-2764
akorobkov-nvda Jul 9, 2026
60a2192
feat(devspace): k3s and machine-a-tron namespace support
akorobkov-nvda Jul 9, 2026
dfff1ac
feat(ci): machine-a-tron container build
akorobkov-nvda Jul 9, 2026
5822760
chore(format): rbac rules
akorobkov-nvda Jul 9, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -728,6 +728,35 @@ jobs:
${{ needs.prepare.outputs.image_registry }}/nvmetal-carbide@${{ needs.build-release-container-x86_64.outputs.image_digest }} \
${{ needs.prepare.outputs.image_registry }}/nvmetal-carbide-aarch64@${{ needs.build-release-container-aarch64.outputs.image_digest }}

build-machine-a-tron:
if: >-
${{
always()
&& github.event_name != 'schedule'
&& needs.prepare.result == 'success'
&& needs.prepare.outputs.source_files_changed == 'true'
}}
needs:
- prepare
uses: ./.github/workflows/docker-build.yml
with:
dockerfile_path: crates/machine-a-tron/Dockerfile
context_path: .
build_args: >-
${{ format('{{"VERSION":"{0}","CI_COMMIT_SHORT_SHA":"{1}"}}',
needs.prepare.outputs.version,
needs.prepare.outputs.short_sha) }}
image_name: ${{ needs.prepare.outputs.image_registry }}/machine-a-tron
image_tag: ${{ needs.prepare.outputs.version }}
platforms: linux/amd64
runner: linux-amd64-cpu16
push: ${{ needs.prepare.outputs.publish_images == 'true' }}
load: false
scan: true
tag_latest: false
timeout_minutes: 60
secrets: inherit

test-release-container-services:
if: >-
${{
Expand Down Expand Up @@ -1716,6 +1745,7 @@ jobs:
- build-artifacts-container-aarch64
- build-release-container-x86_64
- build-release-container-aarch64
- build-machine-a-tron
- test-release-container-services
- build-boot-artifacts-x86
- build-boot-artifacts-bfb
Expand Down Expand Up @@ -1816,6 +1846,7 @@ jobs:
runs-on: linux-amd64-cpu4
needs:
- build-release-container-x86_64
- build-machine-a-tron
- build-forge-cli-x86_64
- build-release-artifacts-x86-host
- build-release-artifacts-arm-host
Expand Down Expand Up @@ -1846,6 +1877,7 @@ jobs:
- build-artifacts-container-aarch64
- build-release-container-x86_64
- build-release-container-aarch64
- build-machine-a-tron
- test-release-container-services
- build-boot-artifacts-x86
- build-boot-artifacts-bfb
Expand Down
5 changes: 4 additions & 1 deletion crates/api-core/src/auth/internal_rbac_rules.rs
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,10 @@ impl InternalRBACRules {
x.perm("UpdateInstancePhoneHomeLastContact", vec![Agent]);
x.perm("SetHostUefiPassword", vec![ForgeAdminCLI]);
x.perm("ClearHostUefiPassword", vec![ForgeAdminCLI]);
x.perm("AddExpectedMachine", vec![ForgeAdminCLI, SiteAgent, Flow]);
x.perm(
"AddExpectedMachine",
vec![ForgeAdminCLI, SiteAgent, Flow, Machineatron],
);
x.perm("DeleteExpectedMachine", vec![ForgeAdminCLI, SiteAgent]);
x.perm("UpdateExpectedMachine", vec![ForgeAdminCLI, SiteAgent]);
x.perm("CreateExpectedMachines", vec![ForgeAdminCLI, SiteAgent]);
Expand Down
89 changes: 89 additions & 0 deletions crates/machine-a-tron/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
# syntax=docker/dockerfile:1.7
#
# Cross-compilation: the BUILDER always runs NATIVE on the build host
# ($BUILDPLATFORM) and cross-compiles to x86_64-unknown-linux-gnu. This is not
# a Mac convenience: building the amd64 stage the naive way on an Apple
# Silicon host runs rustc under QEMU emulation, and QEMU SEGFAULTs assembling
# aws-lc-sys's hand-written x86-64 assembly (s2n-bignum) — even though the
# resulting binary runs fine on real x86_64. On an x86_64 build host the
# builder is simply native and the "cross" compile is a no-op.
#
# Build from the repo root:
#
# docker buildx build \
# --platform linux/amd64 \
# -f crates/machine-a-tron/Dockerfile \
# --build-arg VERSION=0.1.0 \
# -t machine-a-tron:latest \
# .
#
ARG RUST_VERSION=1.96.0

FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-bookworm AS builder

ARG VERSION
ARG CI_COMMIT_SHORT_SHA
ENV VERSION=${VERSION}
ENV CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA}

RUN apt-get update && apt-get install -y --no-install-recommends \
git \
pkg-config \
protobuf-compiler \
libprotobuf-dev \
&& if [ "$(uname -m)" != "x86_64" ]; then \
apt-get install -y --no-install-recommends \
gcc-x86-64-linux-gnu libc6-dev-amd64-cross; \
else \
apt-get install -y --no-install-recommends gcc libc6-dev; \
fi \
&& rm -rf /var/lib/apt/lists/*

RUN rustup target add x86_64-unknown-linux-gnu

ENV CARGO_HOME=/cargo-home
ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
ENV CARGO_TARGET_DIR=/cargo-target
ENV RUST_BACKTRACE=1

WORKDIR /workspace
COPY . .

# Build release binary with cross-compilation
RUN --mount=type=cache,id=nico-mat-cross-cargo-home,target=/cargo-home,sharing=locked \
--mount=type=cache,id=nico-mat-cross-cargo-target,target=/cargo-target,sharing=locked \
if [ "$(uname -m)" != "x86_64" ]; then \
export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \
CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc; \
fi && \
cargo build -p carbide-machine-a-tron --bin machine-a-tron --locked --release \
--target x86_64-unknown-linux-gnu && \
mkdir -p /artifacts && \
cp /cargo-target/x86_64-unknown-linux-gnu/release/machine-a-tron /artifacts/machine-a-tron

# Runtime image
FROM --platform=linux/amd64 debian:bookworm-slim

ARG VERSION
ARG CI_COMMIT_SHORT_SHA
LABEL org.opencontainers.image.version="${VERSION}"
LABEL org.opencontainers.image.revision="${CI_COMMIT_SHORT_SHA}"
LABEL org.opencontainers.image.title="machine-a-tron"
LABEL org.opencontainers.image.description="NICo Hardware Simulation Tool"
LABEL org.opencontainers.image.vendor="NVIDIA"

RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
iproute2 \
iputils-ping \
libssl3 \
libudev1 \
&& rm -rf /var/lib/apt/lists/*

RUN mkdir -p /opt/machine-a-tron/bin /opt/machine-a-tron/templates /tmp/machine-a-tron-data

COPY --from=builder /artifacts/machine-a-tron /opt/machine-a-tron/bin/machine-a-tron
COPY crates/machine-a-tron/templates /opt/machine-a-tron/templates

WORKDIR /opt/machine-a-tron
ENTRYPOINT ["/opt/machine-a-tron/bin/machine-a-tron"]
15 changes: 9 additions & 6 deletions dev/deployment/devspace/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,17 +86,16 @@ devspace deploy
DevSpace will:

- build the local runtime images from [`Dockerfile.api`](Dockerfile.api), [`Dockerfile.bmc-proxy`](Dockerfile.bmc-proxy), and [`Dockerfile.machine-a-tron`](Dockerfile.machine-a-tron)
- deploy the Helm chart in [`helm/`](../../../helm)
- apply the local-only `machine-a-tron` Kubernetes objects from [`machine-a-tron.yaml`](machine-a-tron.yaml) with `kubectl`
- inject the built image names and DevSpace-generated tags into both deployments at runtime
- deploy the Helm chart in [`helm/`](../../../helm) (including `nico-machine-a-tron`)
- inject the built image names and DevSpace-generated tags into all deployments at runtime

The image builds are configured in [`devspace.yaml`](../../../devspace.yaml). The Dockerfiles are multi-stage builds: the builder stage compiles the Rust binary inside Docker from the local `build-container-localdev` image, and the runtime stage copies only the finished binary and required runtime assets. DevSpace first checks whether `build-container-localdev` already exists locally and reuses it if present; otherwise it builds it from [`dev/docker/Dockerfile.build-container-x86_64`](../../../dev/docker/Dockerfile.build-container-x86_64). BuildKit cache mounts are used for Cargo registry, Cargo git checkouts, and Cargo target output so rebuilds stay fast without copying host build artifacts into the image.

The DevSpace images also use Dockerfile-specific ignore files: [`Dockerfile.api.dockerignore`](Dockerfile.api.dockerignore), [`Dockerfile.bmc-proxy.dockerignore`](Dockerfile.bmc-proxy.dockerignore), and [`Dockerfile.machine-a-tron.dockerignore`](Dockerfile.machine-a-tron.dockerignore). This keeps the top-level [`.dockerignore`](../../../.dockerignore) aligned with the main branch for CI and release builds, while still giving the local DevSpace builds a small Docker context.

DevSpace watches the Rust workspace, toolchain metadata, and the runtime Dockerfiles to decide when images need rebuilding.

The production Helm chart is still only responsible for the product services. `machine-a-tron` is deployed separately as plain local-only Kubernetes objects in [`machine-a-tron.yaml`](machine-a-tron.yaml), with DevSpace wiring in the local image tag and certificate issuer from [`devspace.yaml`](../../../devspace.yaml). The local API and BMC proxy configs in [`values.base.yaml`](values.base.yaml) point BMC traffic at `machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266`.
The `nico-machine-a-tron` Helm subchart configuration is in [`values.base.yaml`](values.base.yaml). The local API and BMC proxy configs also point BMC traffic at `nico-machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266`.

Common usage:

Expand All @@ -117,7 +116,11 @@ docker build -t "nico-bmc-proxy:<devspace-generated-tag>" -f dev/deployment/devs
docker build -t "machine-a-tron:<devspace-generated-tag>" -f dev/deployment/devspace/Dockerfile.machine-a-tron .
```

DevSpace then deploys the Helm chart with the built `nico-api` image wired into `global.image.repository` and `global.image.tag`, the built `nico-bmc-proxy` image wired into the `nico-bmc-proxy` chart values, and applies the local-only `machine-a-tron` manifest with its image wired into the `Deployment` spec.
DevSpace then deploys the Helm chart with:
- the built `nico-api` image wired into `global.image.repository` and `global.image.tag`
- the built `nico-bmc-proxy` image wired into the `nico-bmc-proxy` chart values
- the built `machine-a-tron` image wired into the `nico-machine-a-tron` chart values
- certificate issuer settings from the DevSpace environment variables

## Re-initializing infra-controller to a clean slate

Expand All @@ -129,7 +132,7 @@ You can start over again (purging the resources from k8s) by running:
devspace purge -n nico-system
```

and it will delete the NICo Helm release and machine-a-tron deployments.
and it will delete the NICo Helm release (including machine-a-tron).

To clear out the nico database to start from scratch again, run the nuke-postgres.sh helper script:

Expand Down
140 changes: 0 additions & 140 deletions dev/deployment/devspace/machine-a-tron.yaml

This file was deleted.

15 changes: 15 additions & 0 deletions dev/deployment/devspace/values.base.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,21 @@ nico-bmc-proxy:
client_cert = "/var/run/secrets/spiffe.io/tls.crt"
client_key = "/var/run/secrets/spiffe.io/tls.key"

nico-machine-a-tron:
enabled: true
namespaceOverride: "nico-mat"
machineATron:
nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079"
pods:
default:
machines:
config:
hwType: wiwynn_gb200_nvl
hostCount: 10
dpuPerHostCount: 2
oobDhcpRelayAddress: "192.168.192.1"
adminDhcpRelayAddress: "192.168.176.1"

nico-dhcp:
enabled: false
nico-dns:
Expand Down
Loading