From e42ddcbd3349ceb29e89db41d34ee3f67539a87d Mon Sep 17 00:00:00 2001
From: Blake Devcich <blake.devcich@hpe.com>
Date: Mon, 28 Jul 2025 11:28:07 -0500
Subject: [PATCH] Add ssh-container example image

Signed-off-by: Blake Devcich <blake.devcich@hpe.com>
---
 .github/workflows/ssh-container.yml | 111 +++++++++++++++++++++++
 ssh-container/.gitignore            |   2 +
 ssh-container/Dockerfile            |  61 +++++++++++++
 ssh-container/Makefile              | 132 ++++++++++++++++++++++++++++
 ssh-container/README.md             |  96 ++++++++++++++++++++
 ssh-container/entrypoint.sh         |  16 ++++
 ssh-container/ssh-container.yaml    |  27 ++++++
 7 files changed, 445 insertions(+)
 create mode 100644 .github/workflows/ssh-container.yml
 create mode 100644 ssh-container/.gitignore
 create mode 100644 ssh-container/Dockerfile
 create mode 100644 ssh-container/Makefile
 create mode 100644 ssh-container/README.md
 create mode 100644 ssh-container/entrypoint.sh
 create mode 100644 ssh-container/ssh-container.yaml

diff --git a/.github/workflows/ssh-container.yml b/.github/workflows/ssh-container.yml
new file mode 100644
index 0000000..dcf615a
--- /dev/null
+++ b/.github/workflows/ssh-container.yml
@@ -0,0 +1,111 @@
+name: SSH Container build and push
+
+on: [push]
+
+env:
+  # TEST_TARGET: Name of the testing target in the Dockerfile
+  TEST_TARGET: testing
+
+  # DO_TEST - true to build and run unit tests, false to skip the tests
+  DO_TEST: false
+
+  # DO_PUSH - true to push to the HPE_DEPLOY_REPO, false to not push
+  DO_PUSH: true
+
+  # Container build arguments - tie these to Blake for testing
+  USER_UID: 1060
+  USER_GID: 100
+
+  # Image name for the SSH container
+  IMAGE_NAME: ssh-container
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0 
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: Lowercase repository name for docker build
+      id: lowercase-repository-name
+      run: |
+        echo "REPO_NAME=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
+
+    - name: "Set tags for main/master"
+      id: set_tags
+      run: |
+        echo "VERSION_TAG=$(./git-version-gen | grep -v UNKNOWN)" >> ${GITHUB_ENV}
+        echo "TEST_TAG=$(git rev-parse HEAD)-test" >> ${GITHUB_ENV}
+        echo "SHA_TAG=$(git rev-parse HEAD)" >> ${GITHUB_ENV}
+        echo "${GITHUB_ENV}:"
+        cat ${GITHUB_ENV}
+      shell: bash
+
+    - name: "Docker metadata"
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: |
+          ghcr.io/${{ env.REPO_NAME }}-ssh-container
+        tags: |
+          # For merge to master branch, tag example: 'master'
+          type=ref,event=branch
+          # For PR event, tag example: 'pr-3'
+          type=ref,event=pr
+          # For PR event or merge event, tag example: 1.0.1.12-5667
+          type=raw,value=${{ env.VERSION_TAG }}
+          # For PR event or merge, tag example: 566769e04d2436cf5f42ae4f46092c7dff6e668e
+          type=raw,value=${{ env.SHA_TAG }}
+          # For push to semver tag, tag example: 1.0.2
+          # This also sets 'latest'.
+          type=semver,pattern={{version}}
+          # For push to semver tag, tag example: 1.0
+          type=semver,pattern={{major}}.{{minor}}
+
+    - name: Docker login
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Generate SSH keys for container build
+      run: |
+        cd ssh-container
+        make ssh-keys client-keys
+
+    - name: Build the test Docker image
+      if: ${{ env.DO_TEST == 'true' }}
+      id: docker_build_test_target
+      uses: docker/build-push-action@v6
+      with:
+        context: ./ssh-container
+        push: false
+        target: ${{ env.TEST_TARGET }}
+        tags: ${{ env.IMAGE_NAME }}:${{ env.TEST_TAG }}
+        build-args: |
+          USER_UID=${{ env.USER_UID }}
+          USER_GID=${{ env.USER_GID }}
+
+    - name: Run the Docker image unit tests
+      if: ${{ env.DO_TEST == 'true' }}
+      run: docker run ${{ env.IMAGE_NAME }}:${{ env.TEST_TAG }}
+
+    - name: Build the final Docker image
+      id: docker_build
+      uses: docker/build-push-action@v6
+      with:
+        context: ./ssh-container
+        push: false
+        tags: ${{ steps.meta.outputs.tags }}
+        build-args: |
+          USER_UID=${{ env.USER_UID }}
+          USER_GID=${{ env.USER_GID }}
+
+    - name: Peek at the docker images
+      run: docker images | grep ${{ env.IMAGE_NAME }}
+
+
diff --git a/ssh-container/.gitignore b/ssh-container/.gitignore
new file mode 100644
index 0000000..e33a44a
--- /dev/null
+++ b/ssh-container/.gitignore
@@ -0,0 +1,2 @@
+ssh_client_keys
+ssh_host_keys
\ No newline at end of file
diff --git a/ssh-container/Dockerfile b/ssh-container/Dockerfile
new file mode 100644
index 0000000..52caef5
--- /dev/null
+++ b/ssh-container/Dockerfile
@@ -0,0 +1,61 @@
+FROM ghcr.io/nearnodeflash/nnf-dm-copy-offload:0.1.19
+
+# Build arguments for user UID/GID
+ARG USER_UID=1234
+ARG USER_GID=1234
+
+# Set environment variables
+ENV USER_UID=${USER_UID}
+ENV USER_GID=${USER_GID}
+
+ENV SESSION_DIR="/home/mpiuser/ssh-session"
+ENV KEYS_DIR="/home/mpiuser/ssh-session/keys"
+
+# Modify existing mpiuser to use specified UID/GID
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN TARGET_GROUP=$(getent group ${USER_GID} | cut -d: -f1) && \
+    if [ -z "$TARGET_GROUP" ]; then \
+        groupmod -g ${USER_GID} mpiuser; \
+        TARGET_GROUP="mpiuser"; \
+    fi && \
+    usermod -u ${USER_UID} -g ${USER_GID} mpiuser
+SHELL ["/bin/sh", "-c"]
+
+# Set up SSH directory for mpiuser
+RUN mkdir -p /home/mpiuser/.ssh && \
+    mkdir -p ${SESSION_DIR} && \
+    mkdir -p ${KEYS_DIR}
+
+# Copy SSH host keys
+COPY ssh_host_keys/ssh_host_rsa_key ${KEYS_DIR}/ssh_host_rsa_key
+COPY ssh_host_keys/ssh_host_rsa_key.pub ${KEYS_DIR}/ssh_host_rsa_key.pub
+
+# Copy client public key for authorized access
+COPY ssh_client_keys/mpiuser_key.pub ${KEYS_DIR}mpiuser_key.pub
+RUN cat ${KEYS_DIR}mpiuser_key.pub > ${KEYS_DIR}/authorized_keys
+
+# Override any entrypoint from base image and set our own
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod a+x /entrypoint.sh
+
+WORKDIR /home/mpiuser
+
+# Configure SSH - port is set at runtime so we don't hardcode it here
+RUN echo "HostKey $KEYS_DIR/ssh_host_rsa_key" > "$SESSION_DIR/sshd_config" && \
+    echo "AuthorizedKeysFile $KEYS_DIR/authorized_keys" >> "$SESSION_DIR/sshd_config" && \
+    echo "PermitRootLogin no" >> "$SESSION_DIR/sshd_config" && \
+    echo "PasswordAuthentication no" >> "$SESSION_DIR/sshd_config"
+
+# Set proper permissions on keys
+RUN chmod 600 ${KEYS_DIR}/ssh_host_rsa_key && \
+    chmod 644 ${KEYS_DIR}/ssh_host_rsa_key.pub && \
+    chmod 600 ${KEYS_DIR}/authorized_keys && \
+    chmod 700 /home/mpiuser/.ssh && \
+    chown -R ${USER_UID}:${USER_GID} /home/mpiuser
+
+# Labels for identifying the UID/GID
+LABEL uid="${USER_UID}"
+LABEL gid="${USER_GID}"
+LABEL description="SSH container with mpiuser (UID:${USER_UID}, GID:${USER_GID})"
+
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/ssh-container/Makefile b/ssh-container/Makefile
new file mode 100644
index 0000000..4664c3e
--- /dev/null
+++ b/ssh-container/Makefile
@@ -0,0 +1,132 @@
+# TODO: Set the UID and GID To match the user who is going to use this container. Due to security, the UID and GID
+# *must* be baked in to the image in order for sshd to obtain the correct permissions to enable ssh security.
+UID ?= 1060
+GID ?= 100
+
+# NOTE: SSH_PORT is only for local testing purposes. In Rabbit User Containers, the port is assigned dynamically via the
+# NNF_CONTAINER_PORTS environment variable.
+SSH_PORT ?= 5123
+
+# SSH key files - these are generated locally and used in the container. The mpiuser_key is used for ssh authentication
+# to connect to the container. Do not lose the client keys.
+SSH_HOST_KEY_DIR = ./ssh_host_keys
+SSH_HOST_RSA_KEY = $(SSH_HOST_KEY_DIR)/ssh_host_rsa_key
+SSH_CLIENT_KEY_DIR = ./ssh_client_keys
+SSH_CLIENT_KEY = $(SSH_CLIENT_KEY_DIR)/mpiuser_key
+
+# Image name and tag
+IMAGE_NAME ?= ssh-container
+TAG ?= latest
+
+# CONTAINER_TOOL defines the container tool to be used for building images.
+# Be aware that the target commands are only tested with Docker which is
+# scaffolded by default. However, you might want to replace it to use other
+# tools. (i.e. podman)
+CONTAINER_TOOL ?= docker
+
+# Container name for easier management
+CONTAINER_NAME = ssh-container-$(UID)-$(GID)
+
+.PHONY: all docker-build clean ssh-keys client-keys keys run stop connect status help
+
+all: docker-build
+
+help: ## Show this help message
+	@echo "Available targets:"
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  %-15s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+# Generate SSH host keys
+ssh-keys: ## Generate SSH host keys for the container
+	@echo "Generating SSH host keys..."
+	@mkdir -p $(SSH_HOST_KEY_DIR)
+	@if [ ! -f $(SSH_HOST_RSA_KEY) ]; then \
+		ssh-keygen -t rsa -b 4096 -f $(SSH_HOST_RSA_KEY) -N '' -C "SSH host RSA key"; \
+	fi
+	@echo "SSH host keys generated successfully"
+
+# Generate SSH client keys for authentication
+client-keys: ## Generate SSH client keys for authentication
+	@echo "Generating SSH client keys..."
+	@mkdir -p $(SSH_CLIENT_KEY_DIR)
+	@if [ ! -f $(SSH_CLIENT_KEY) ]; then \
+		ssh-keygen -t rsa -b 4096 -f $(SSH_CLIENT_KEY) -N '' -C "mpiuser@ssh-container"; \
+		echo ""; \
+		echo "Client key generated: $(SSH_CLIENT_KEY)"; \
+		echo "Public key: $(SSH_CLIENT_KEY).pub"; \
+		echo ""; \
+		echo "To connect without password:"; \
+		echo "  ssh -i $(SSH_CLIENT_KEY) -p $(SSH_PORT) mpiuser@localhost"; \
+		echo ""; \
+	fi
+
+# Generate both SSH host and client keys
+keys: ssh-keys client-keys ## Generate both SSH host and client keys
+	@echo "SSH keys generated successfully"
+	@echo "Host keys in: $(SSH_HOST_KEY_DIR)"
+	@echo "Client keys in: $(SSH_CLIENT_KEY_DIR)"
+
+# Build the container image
+docker-build:
+	@echo "Building $(CONTAINER_TOOL) image with UID=$(UID), GID=$(GID)..."
+	$(CONTAINER_TOOL) build \
+		--platform linux/amd64 \
+		--build-arg USER_UID=$(UID) \
+		--build-arg USER_GID=$(GID) \
+		-t $(IMAGE_NAME):$(TAG) \
+		-t $(IMAGE_NAME):uid$(UID)-gid$(GID) \
+		.
+	@echo "$(CONTAINER_TOOL) image built successfully"
+	@echo "Image tags:"
+	@echo "  - $(IMAGE_NAME):$(TAG)"
+	@echo "  - $(IMAGE_NAME):uid$(UID)-gid$(GID)"
+	@echo ""
+	@echo "To inspect UID/GID/PORT labels:"
+	@echo "  $(CONTAINER_TOOL) inspect $(IMAGE_NAME):$(TAG) | grep -A 10 Labels"
+
+# Run the container
+run: docker-build ## Run the container with SSH service (as UID 1060 and GID 100)
+	$(CONTAINER_TOOL) run -d \
+		--platform linux/amd64 \
+		--name $(CONTAINER_NAME) \
+		--user $(UID):$(GID) \
+		-p $(SSH_PORT):$(SSH_PORT) \
+		-e NNF_CONTAINER_PORTS=$(SSH_PORT) \
+		$(IMAGE_NAME):$(TAG)
+	@echo "Container started with SSH on port $(SSH_PORT) (accessible via host port $(SSH_PORT))"
+	@echo "Connect with: make connect"
+
+# Stop and remove container
+stop: ## Stop and remove the container
+	-$(CONTAINER_TOOL) stop $(CONTAINER_NAME)
+	-$(CONTAINER_TOOL) rm $(CONTAINER_NAME)
+
+# Check container status
+status: ## Check if the container is running
+	@if $(CONTAINER_TOOL) ps | grep -q $(CONTAINER_NAME); then \
+		echo "Container $(CONTAINER_NAME) is running"; \
+		$(CONTAINER_TOOL) ps | grep $(CONTAINER_NAME); \
+	else \
+		echo "Container $(CONTAINER_NAME) is not running"; \
+	fi
+
+# Connect to the running container using SSH keys
+connect: ## Connect to the container via SSH
+	@if [ ! -f $(SSH_CLIENT_KEY) ]; then \
+		echo "Client key not found. Run 'make client-keys' first."; \
+		exit 1; \
+	fi
+	@if ! $(CONTAINER_TOOL) ps | grep -q $(CONTAINER_NAME); then \
+		echo "Container $(CONTAINER_NAME) is not running."; \
+		echo "Run 'make run' to start the container first."; \
+		echo "Or run 'make status' to check container status."; \
+		exit 1; \
+	fi
+	@echo "Connecting to SSH container..."
+	ssh -i $(SSH_CLIENT_KEY) -p $(SSH_PORT) -o StrictHostKeyChecking=no mpiuser@localhost
+
+# Clean up generated files and containers
+clean: stop ## Clean up generated files and containers
+	rm -rf $(SSH_HOST_KEY_DIR)
+	rm -rf $(SSH_CLIENT_KEY_DIR)
+	-$(CONTAINER_TOOL) rmi $(IMAGE_NAME):$(TAG)
+	-$(CONTAINER_TOOL) rmi $(IMAGE_NAME):uid$(UID)-gid$(GID)
\ No newline at end of file
diff --git a/ssh-container/README.md b/ssh-container/README.md
new file mode 100644
index 0000000..d8df5b7
--- /dev/null
+++ b/ssh-container/README.md
@@ -0,0 +1,96 @@
+# SSH Container for Rabbit User Container Workflows
+
+This container provides SSH access to launcher containers running on Rabbit nodes.
+
+## Overview
+
+The SSH container enables remote access to Rabbit User Container workflows. It requires predetermined UID/GID and
+generated SSH keys baked into the image for secure authentication. This means users will need to build their own image
+and publish it somewhere. 
+
+⚠️ **Security Note**: Images are user-specific and the generated keys must be kept secure - loss of keys means loss of
+access to the container.
+
+## Building and Publishing Your Image
+
+In this example, I use ghcr.io to publish the image publicly. Change this to an internal registry. This is using docker, but podman can be used.
+
+```bash
+# 1. Generate SSH keys
+make keys
+
+# 2. Build with your UID/GID  
+make build UID=1060 GID=100 # you can also change this in the Makefile
+
+# 3. Test locally
+make run
+make connect
+
+# 4. Publish to your registry
+docker tag ssh-container:latest ghcr.io/$USERNAME/ssh-container:latest
+docker push ghcr.io/$USERNAME/ssh-container:latest
+```
+
+The keys will be created and kept in `ssh_client_keys/`. The private key `mpiuser_key` is required to SSH into the
+container — **keep this file safe and secure; if you lose it, you will lose access to the container.**
+
+Only the private key (`mpiuser_key`) is needed on the compute node where you SSH from. The corresponding public key is
+already included in the container image.
+
+## NNF Container Profile
+
+Edit the `NnfContainerProfile` to point to your image. An example is provided in this repo in `ssh-container.yaml`. Have
+an admin create a profile referencing your published image:
+
+```shell
+kubectl apply -f ssh-container.yaml
+```
+## Run 
+
+Once you have an image published and available to you via an `NnfContainerProfile`, you can use a flux allocation to
+start the containers and then jump to a compute node. As noted previously, you will need your private ssh key to get in.
+
+Start the allocation and use a `container` directive:
+
+```shell
+flux alloc -N4 --setattr=dw="#DW container name=ssh-container profile=ssh-container"
+```
+
+Then, ssh into the container using the private key:
+
+```shell
+[blake@rabbit-compute-2 ~]$ ssh -i ./mpiuser_key -p $NNF_CONTAINER_PORTS mpiuser@$NNF_CONTAINER_LAUNCHER
+
+The programs included with the Debian GNU/Linux system are free software;
+the exact distribution terms for each program are described in the
+individual files in /usr/share/doc/*/copyright.
+
+Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent
+permitted by applicable law.
+
+$ hostname
+fluxjob-321103304048246784-launcher
+$ whoami
+mpiuser$
+```
+
+Once inside the container, you can then run mpirun commands:
+
+```
+$ mpirun hostname
+fluxjob-321103304048246784-worker-0
+fluxjob-321103304048246784-worker-1
+```
+
+In this case, we have 2 workers on 2 rabbit nodes.
+
+## Customization
+
+Add tools by modifying the Dockerfile:
+
+```dockerfile
+# Add your tools after the base setup
+RUN apt-get update && apt-get install -y \
+    your-tool \
+    && rm -rf /var/lib/apt/lists/*
+```
\ No newline at end of file
diff --git a/ssh-container/entrypoint.sh b/ssh-container/entrypoint.sh
new file mode 100644
index 0000000..3ec1812
--- /dev/null
+++ b/ssh-container/entrypoint.sh
@@ -0,0 +1,16 @@
+#!/bin/sh 
+
+echo "Starting SSH container entrypoint script..."
+
+whoami
+id
+
+ls -la /home/mpiuser
+ls -la /home/mpiuser/ssh-session
+
+export -p > /home/mpiuser/ssh-session/env.sh
+echo "[ -f /home/mpiuser/ssh-session/env.sh ] && . /home/mpiuser/ssh-session/env.sh" >> "$HOME"/.profile
+
+echo "Port $NNF_CONTAINER_PORTS" >> /home/mpiuser/ssh-session/sshd_config && \
+
+/usr/sbin/sshd -D -f "/home/mpiuser/ssh-session/sshd_config" -E "/home/mpiuser/ssh-session/sshd.log"
\ No newline at end of file
diff --git a/ssh-container/ssh-container.yaml b/ssh-container/ssh-container.yaml
new file mode 100644
index 0000000..e1205d2
--- /dev/null
+++ b/ssh-container/ssh-container.yaml
@@ -0,0 +1,27 @@
+apiVersion: nnf.cray.hpe.com/v1alpha7
+kind: NnfContainerProfile
+metadata:
+  name: ssh-container
+  namespace: nnf-system
+data:
+  mpiSpec:
+    launcher:
+      containers:
+        - image: ghcr.io/bdevcich/ssh-container:latest
+          imagePullPolicy: IfNotPresent
+          name: ssh-svr
+    worker:
+      containers:
+        - image: ghcr.io/bdevcich/ssh-container:latest
+          imagePullPolicy: IfNotPresent
+          name: ssh-worker
+  numPorts: 1
+  pinned: false
+  postRunTimeoutSeconds: 0
+  preRunTimeoutSeconds: 300
+  retryLimit: 2
+  storages:
+    - name: DW_JOB_my_storage
+      optional: true
+    - name: DW_GLOBAL_lus
+      optional: true