feat: update docker for remote build and push (#115)

This PR enables the container to run the nix build command as a non root
user. This functionality is required to allow the builder to run on
hfjobs; which sets the user is 1000.

deploy a build job via

```bash
hfjobs run \
    drbh/kernel-builder-dev:job-runner-0 \
    /home/nixuser/bin/cli.sh \
    fetch \
    https://huggingface.co/kernels-community/activation.git
```

notes
- `drbh/kernel-builder-dev:job-runner-0` is an image of the attached
Dockerfile
- `/home/nixuser/bin/cli.sh` is the entrypoint that needs to be
specified on `hfjobs`
- `fetch` is the docker command
- `https://huggingface.co/kernels-community/activation.git` is the
kernel repo to build

currently this feature does not push the built files to the hub but that
can/should be added in a followup PR when a suitable upload strategy is
defined.

Author: drbh

Dockerfile

@@ -1,18 +1,44 @@
 FROM nixos/nix:2.18.8
+
 # default build args
-ARG MAX_JOBS=4
-ARG CORES=4
+ARG MAX_JOBS=1
+ARG CORES=1
+
+# Set up Nix configuration and user
 RUN echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf \
     && echo "max-jobs = $MAX_JOBS" >> /etc/nix/nix.conf \
     && echo "cores = $CORES" >> /etc/nix/nix.conf \
-    && nix profile install nixpkgs#cachix nixpkgs#git-lfs \
-    && cachix use kernel-builder
-WORKDIR /kernelcode
-COPY . /etc/kernel-builder/
+    && echo "trusted-users = root nixuser" >> /etc/nix/nix.conf \
+    # Create user entries directly in password and group files
+    && echo "nixuser:x:1000:1000:NixOS User:/home/nixuser:/bin/bash" >> /etc/passwd \
+    && echo "nixuser:x:1000:" >> /etc/group \
+    && mkdir -p /home/nixuser/kernelcode \
+    # Create Nix directories with proper permissions
+    && mkdir -p /nix/var/nix/profiles/per-user/nixuser \
+    && mkdir -p /nix/var/nix/gcroots/per-user/nixuser \
+    && chown -R 1000:1000 /home/nixuser /nix/var/nix/profiles/per-user/nixuser /nix/var/nix/gcroots/per-user/nixuser \
+    # Install necessary packages
+    && nix profile install nixpkgs#cachix nixpkgs#git-lfs nixpkgs#gawk \
+    && cachix use kernel-builder 
+
+# Set permissions for Nix directories
+RUN chown -R nixuser:nixuser /nix
+
+# Set working directory and copy files
+WORKDIR /home/nixuser/kernelcode
+COPY --chown=nixuser:nixuser . /home/nixuser/kernel-builder/
+
+# Set environment variables
 ENV MAX_JOBS=${MAX_JOBS}
 ENV CORES=${CORES}
-RUN mkdir -p /etc/kernelcode && \
-    cat <<'EOF' > /etc/kernelcode/cli.sh
+ENV HF_TOKEN=${HF_TOKEN}
+ENV HOME=/home/nixuser
+ENV PUSH_REVISION=hfjob-build
+ENV REPO=kernels-community/job-build-test-repo
+
+# Set up CLI script in nixuser's home
+RUN mkdir -p /home/nixuser/bin && \
+    cat <<'EOF' > /home/nixuser/bin/cli.sh
 #!/bin/sh
 set -e
 
@@ -38,23 +64,23 @@ function show_usage {
   echo "  --cores, -c NUMBER  Set number of cores per job (default: $CORES)"
   echo ""
   echo "Examples:"
-  echo "  docker run -v \$(pwd):/kernelcode kernel-builder:dev build"
-  echo "  docker run -it -v \$(pwd):/kernelcode kernel-builder:dev dev"
+  echo "  docker run -v \$(pwd):/home/nixuser/kernelcode kernel-builder:dev build"
+  echo "  docker run -it -v \$(pwd):/home/nixuser/kernelcode kernel-builder:dev dev"
   echo "  docker run kernel-builder:dev fetch https://huggingface.co/user/repo.git"
 }
 
 # Function to generate a basic flake.nix if it doesn't exist
 function ensure_flake_exists {
-  if [ ! -f "/kernelcode/flake.nix" ]; then
+  if [ ! -f "/home/nixuser/kernelcode/flake.nix" ]; then
     echo "No flake.nix found, creating a basic one..."
-    cat <<'FLAKE_EOF' > /kernelcode/flake.nix
+    cat <<'FLAKE_EOF' > /home/nixuser/kernelcode/flake.nix
 {
   description = "Flake for Torch kernel extension";
 
   inputs = {
     kernel-builder.url = "github:huggingface/kernel-builder";
   };
-
+  
   outputs = { self, kernel-builder, }:
     kernel-builder.lib.genFlakeOutputs {
       path = ./.;
@@ -72,9 +98,9 @@ FLAKE_EOF
 function build_extension {
   echo "Building Torch Extension Bundle"
   # Check if kernelcode is a git repo and get hash if possible
-  if [ -d "/kernelcode/.git" ]; then
+  if [ -d "/home/nixuser/kernelcode/.git" ]; then
     # Mark git as safe to allow commands
-    git config --global --add safe.directory /kernelcode
+    git config --global --add safe.directory /home/nixuser/kernelcode
     # Try to get git revision
     REV=$(git rev-parse --short=8 HEAD)
 
@@ -92,17 +118,18 @@ function build_extension {
   ensure_flake_exists
 
   # Pure bundle build
+  # TODO: remove the "bundle" after resolving
   echo "Building with Nix..."
   nix build \
-    . \
+    .\#bundle \
     --max-jobs $MAX_JOBS \
     -j $CORES \
-    -L
-  
-  echo "Build completed. Copying results to /kernelcode/build/"
-  mkdir -p /kernelcode/build
-  cp -r --dereference ./result/* /kernelcode/build/
-  chmod -R u+w /kernelcode/build
+    -L 2>&1 | awk '{ print strftime("[%Y-%m-%d %H:%M:%S]"), $0; fflush(); }'
+
+  echo "Build completed. Copying results to /home/nixuser/kernelcode/build/"
+  mkdir -p /home/nixuser/kernelcode/build
+  cp -r --dereference ./result/* /home/nixuser/kernelcode/build/
+  chmod -R u+w /home/nixuser/kernelcode/build
   echo 'Done'
 }
 
@@ -111,7 +138,7 @@ function start_dev_shell {
   echo "Starting development shell..."
   # Check for flake.nix or create one
   ensure_flake_exists
-  /root/.nix-profile/bin/nix develop
+  nix develop
 }
 
 # Function to fetch and build from URL
@@ -123,11 +150,25 @@ function fetch_and_build {
   fi
 
   echo "Fetching code from $1"
-  rm -rf /kernelcode/* /kernelcode/.* 2>/dev/null || true
+  rm -rf /home/nixuser/kernelcode/* /home/nixuser/kernelcode/.* 2>/dev/null || true
   git lfs install
-  git clone "$1" /kernelcode
-  cd /kernelcode
+  git clone "$1" /home/nixuser/kernelcode
+  cd /home/nixuser/kernelcode
   build_extension
+  echo "Build completed. Results are in /home/nixuser/kernelcode/build/"
+  
+  # skip login to huggingface since token is set in the env
+  # check user
+  nix shell nixpkgs#python3 nixpkgs#python3Packages.huggingface-hub -c huggingface-cli whoami
+
+  # upload the build to the repo
+  nix shell nixpkgs#python3 nixpkgs#python3Packages.huggingface-hub -c huggingface-cli \
+    upload \
+    --revision ${PUSH_REVISION} \
+    --commit-message "Build from kernel-builder job" \
+    ${REPO} \
+    /home/nixuser/kernelcode/build/ \
+    build/
 }
 
 # Parse arguments
@@ -182,5 +223,12 @@ case $COMMAND in
 esac
 EOF
 
-RUN chmod +x /etc/kernelcode/cli.sh
-ENTRYPOINT ["/etc/kernelcode/cli.sh"]
+# Set permissions and make the script executable
+RUN chmod +x /home/nixuser/bin/cli.sh && \
+    chown -R nixuser:nixuser /home/nixuser
+
+# Switch to nixuser
+USER nixuser
+
+# Use the cli.sh script directly
+ENTRYPOINT ["/home/nixuser/bin/cli.sh"]

README.md

@@ -32,7 +32,7 @@ cd examples/activation
 
 # then run the following command to build the kernel
 docker run --rm \
-    -v $(pwd):/kernelcode \
+    -v $(pwd):/home/nixuser/kernelcode \
     ghcr.io/huggingface/kernel-builder:latest
 ```
 

docs/docker.md

@@ -27,7 +27,7 @@ cd examples/activation
 
 # then run the following command to build the kernel
 docker run --rm \
-    -v $(pwd):/kernelcode \
+    -v $(pwd):/home/nixuser/kernelcode \
     ghcr.io/huggingface/kernel-builder:latest
 ```
 
@@ -49,10 +49,10 @@ The kernel builder now includes a command-line interface for easier interaction.
 
 ```bash
 # Build the kernel (same as the Quick Start example)
-docker run --rm -v $(pwd):/kernelcode ghcr.io/huggingface/kernel-builder:latest build
+docker run --rm -v $(pwd):/home/nixuser/kernelcode ghcr.io/huggingface/kernel-builder:latest build
 
 # Start an ephemeral development shell
-docker run --rm -it -v $(pwd):/kernelcode ghcr.io/huggingface/kernel-builder:latest dev
+docker run --rm -it -v $(pwd):/home/nixuser/kernelcode ghcr.io/huggingface/kernel-builder:latest dev
 
 # Build from a Git URL
 docker run --rm ghcr.io/huggingface/kernel-builder:latest fetch https://huggingface.co/kernels-community/activation.git
@@ -74,7 +74,7 @@ The kernel builder can be configured in two ways:
 
 ```bash
 docker run --rm \
-    -v $(pwd):/kernelcode \
+    -v $(pwd):/home/nixuser/kernelcode \
     -e MAX_JOBS=8 \
     -e CORES=8 \
     ghcr.io/huggingface/kernel-builder:latest
@@ -91,7 +91,7 @@ You can also specify these parameters using command-line options:
 
 ```bash
 docker run --rm \
-    -v $(pwd):/kernelcode \
+    -v $(pwd):/home/nixuser/kernelcode \
     ghcr.io/huggingface/kernel-builder:latest build --jobs 8 --cores 4
 ```
 
@@ -102,7 +102,7 @@ For development purposes, you can start an interactive shell with:
 ```bash
 docker run -it \
   --name my-dev-env \
-  -v "$(pwd)":/kernelcode \
+  -v "$(pwd)":/home/nixuser/kernelcode \
   ghcr.io/huggingface/kernel-builder:latest dev
 ```
 
@@ -116,7 +116,7 @@ For iterative development, you can create a persistent container to maintain the
 # Create a persistent container and start a development shell
 docker run -it \
   --name my-persistent-dev-env \
-  -v "$(pwd)":/kernelcode \
+  -v "$(pwd)":/home/nixuser/kernelcode \
   ghcr.io/huggingface/kernel-builder:latest dev
 ```
 
@@ -173,7 +173,7 @@ git clone git@hf.co:kernels-community/activation
 cd activation
 # then run the build command
 docker run --rm \
-    -v $(pwd):/kernelcode \
+    -v $(pwd):/home/nixuser/kernelcode \
     ghcr.io/huggingface/kernel-builder:latest
 # we should now have the built kernels on our host
 ls result
@@ -195,7 +195,7 @@ This will clone the repository into the container, build the kernels, and save t
 
 ```bash
 docker run --rm \
-    -v /path/to/output:/kernelcode/build \
+    -v /path/to/output:/home/nixuser/kernelcode/build \
     ghcr.io/huggingface/kernel-builder:latest fetch https://huggingface.co/kernels-community/activation.git
 ```
 
@@ -209,7 +209,7 @@ docker build -t ghcr.io/huggingface/kernel-builder:latest .
 
 # You can build a kernel using this development container:
 cd examples/activation
-docker run --rm -v $(pwd):/kernelcode ghcr.io/huggingface/kernel-builder:latest
+docker run --rm -v $(pwd):/home/nixuser/kernelcode ghcr.io/huggingface/kernel-builder:latest
 
 # copying path '/nix/store/1b79df96k9npmrdgwcljfh3v36f7vazb-source' from 'https://cache.nixos.org'...
 # trace: evaluation warning: CUDA versions older than 12.0 will be removed in Nixpkgs 25.05; see the 24.11 release notes for more information