From e38ed886cc1550993c7d847065cf87235c6178af Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Thu, 16 Jul 2026 15:17:19 +0100 Subject: [PATCH] Add ecflow-all image and docker_ecflow_ui.sh launcher ecflow-all is a new all-in-one Docker image capable of running ecflow_ui against a remote ecFlow server (e.g. on ECMWF's HPC) through a proxychains4 + SSH SOCKS tunnel on the Docker host. docker_ecflow_ui.sh is a self-contained ecflow_ui launcher, which handles all the details of running inside Docker: - resolves the proxy host to a literal IP at container start, - generates the proxychains4.conf - defines DISPLAY export X11 traffic behaves correctly. dockit.yml builds both images from a single package/dockerize job pair, matrixed over preset. --- .github/workflows/dockit.yml | 68 ++++++-- releng/dockit/.gitignore | 2 +- releng/dockit/INSTRUCTIONS.md | 94 +++++++++-- releng/dockit/docker_ecflow_ui.sh | 166 +++++++++++++++++++ releng/dockit/ecflow-all/Dockerfile | 37 +++++ releng/dockit/ecflow-all/launch-ecflow_ui.sh | 49 ++++++ 6 files changed, 387 insertions(+), 29 deletions(-) create mode 100755 releng/dockit/docker_ecflow_ui.sh create mode 100644 releng/dockit/ecflow-all/Dockerfile create mode 100755 releng/dockit/ecflow-all/launch-ecflow_ui.sh diff --git a/.github/workflows/dockit.yml b/.github/workflows/dockit.yml index 283fb291d..0e94030d1 100644 --- a/.github/workflows/dockit.yml +++ b/.github/workflows/dockit.yml @@ -13,7 +13,6 @@ jobs: outputs: timestamp: ${{ steps.timestamp.outputs.timestamp }} - preset: ${{ steps.timestamp.outputs.preset }} jobs: ${{ steps.timestamp.outputs.jobs }} steps: @@ -21,12 +20,13 @@ jobs: id: timestamp run: | echo "timestamp=$(date +'%Y%m%dT%H%M%S')" >> $GITHUB_OUTPUT - echo "preset=linux.gcc.serveronly.relwithdebinfo" >> $GITHUB_OUTPUT echo "jobs=4" >> $GITHUB_OUTPUT - # Create the ecflow package + # + # Create the ecflow package, per image variant + # package: - name: package + name: package (${{ matrix.image }}) needs: bootstrap if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-docker' }} @@ -38,6 +38,21 @@ jobs: timeout-minutes: 60 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + preset: + - linux.gcc.serveronly.relwithdebinfo + - linux.gcc.all.relwithdebinfo + include: + - preset: linux.gcc.serveronly.relwithdebinfo + image: ecflow-serveronly-dev + context: ecflow-server + - preset: linux.gcc.all.relwithdebinfo + image: ecflow-all-dev + context: ecflow-all + outputs: version: ${{ steps.vars.outputs.version }} @@ -88,14 +103,14 @@ jobs: id: configure-ecflow run: | cd ${GITHUB_WORKSPACE}/ecflow - cmake --preset ${{ needs.bootstrap.outputs.preset }} \ + cmake --preset ${{ matrix.preset }} \ -DCUSTOM_DEBIAN_PACKAGE_VERSION=${{ steps.vars.outputs.version }} - name: Build ecflow run: | cd ${GITHUB_WORKSPACE}/ecflow cmake --build \ - --preset ${{ needs.bootstrap.outputs.preset }} \ + --preset ${{ matrix.preset }} \ --jobs ${{ needs.bootstrap.outputs.jobs }} \ --target all @@ -103,33 +118,50 @@ jobs: run: | cd ${GITHUB_WORKSPACE}/ecflow cmake --build \ - --preset ${{ needs.bootstrap.outputs.preset }} \ + --preset ${{ matrix.preset }} \ --target package - name: Upload ecflow debian package uses: actions/upload-artifact@v7 with: - name: ecflow-debian-package - path: ecflow/.deploy/build/${{ needs.bootstrap.outputs.preset }}/ecflow-*.deb + name: ecflow-debian-package-${{ matrix.image }} + path: ecflow/.deploy/build/${{ matrix.preset }}/ecflow-*.deb if-no-files-found: error # - name: Test ecflow # run: | # cd ${{ github.workspace }}/ecflow -# ctest --preset ${{ needs.bootstrap.outputs.preset }} \ +# ctest --preset ${{ matrix.preset }} \ # --jobs ${{ needs.bootstrap.outputs.jobs }} \ # --output-on-failure \ # -L nightly - # Create ecflow docker image + # + # Create the ecflow docker image + # dockerize: - name: dockerize + name: dockerize (${{ matrix.image }}) needs: [ bootstrap, package ] runs-on: ubuntu-latest timeout-minutes: 60 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + preset: + - linux.gcc.serveronly.relwithdebinfo + - linux.gcc.all.relwithdebinfo + include: + - preset: linux.gcc.serveronly.relwithdebinfo + image: ecflow-serveronly-dev + context: ecflow-server + - preset: linux.gcc.all.relwithdebinfo + image: ecflow-all-dev + context: ecflow-all + steps: - name: Checkout 'ecflow' uses: actions/checkout@v4 @@ -139,8 +171,8 @@ jobs: - name: Download ecflow debian package uses: actions/download-artifact@v7 with: - name: ecflow-debian-package - path: ecflow/releng/dockit/ecflow-server + name: ecflow-debian-package-${{ matrix.image }} + path: ecflow/releng/dockit/${{ matrix.context }} - name: Log in to Harbor uses: docker/login-action@v3 @@ -158,16 +190,16 @@ jobs: - name: Build and push Docker image uses: docker/build-push-action@v7 with: - file: ecflow/releng/dockit/ecflow-server/Dockerfile - context: ecflow/releng/dockit/ecflow-server + file: ecflow/releng/dockit/${{ matrix.context }}/Dockerfile + context: ecflow/releng/dockit/${{ matrix.context }} build-args: | ECFLOW_VERSION=${{ needs.package.outputs.version }} ECFLOW_PACKAGE=ecflow-${{ needs.package.outputs.version }}-Linux_x86_64.deb platforms: linux/amd64 push: true tags: | - eccr.ecmwf.int/ecflow-dev-environments/ecflow-serveronly-dev:latest - eccr.ecmwf.int/ecflow-dev-environments/ecflow-serveronly-dev:${{ needs.package.outputs.version }}_${{ needs.bootstrap.outputs.timestamp }} + eccr.ecmwf.int/ecflow-dev-environments/${{ matrix.image }}:latest + eccr.ecmwf.int/ecflow-dev-environments/${{ matrix.image }}:${{ needs.package.outputs.version }}_${{ needs.bootstrap.outputs.timestamp }} labels: | org.opencontainers.image.source=${{ github.repositoryUrl }} org.opencontainers.image.revision=${{ github.sha }} diff --git a/releng/dockit/.gitignore b/releng/dockit/.gitignore index 89c3ce9d3..51cb9f739 100644 --- a/releng/dockit/.gitignore +++ b/releng/dockit/.gitignore @@ -1,2 +1,2 @@ -ecflow-*.deb +ecflow-*/*.deb *.sandbox diff --git a/releng/dockit/INSTRUCTIONS.md b/releng/dockit/INSTRUCTIONS.md index 8f6b0874e..c93ac9645 100644 --- a/releng/dockit/INSTRUCTIONS.md +++ b/releng/dockit/INSTRUCTIONS.md @@ -16,16 +16,17 @@ stated. ## Relationship to the `dockit` GitHub Actions workflow -The action `.github/workflows/dockit.yml` automates the `ecflow-server` image build process described below, -end-to-end, on `workflow_dispatch`: +The action `.github/workflows/dockit.yml` automates both image build processes described below, end-to-end, +on `workflow_dispatch`, as two jobs, each running once per image variant via a build matrix (preset is the +matrix dimension: each leg pairs a preset with the image name and Dockerfile directory it belongs to): -1. The `package` job builds the ecFlow Debian package inside `marcosbento/lumen:debian-13.5`, following the same - checkout/configure/build/package steps as `ecflow-server.build.package.sh`, and uploads the resulting `.deb` - as a build artefact. +1. The `package` job builds the ecFlow Debian package inside `marcosbento/lumen:debian-13.5`, following the + same checkout/configure/build/package steps as `ecflow-server.build.package.sh`, and uploads the resulting + `.deb` as an `ecflow-debian-package-` artefact. -2. The `dockerize` job downloads that artifact into `ecflow-server/` and builds the Docker image from - `ecflow-server/Dockerfile`, passing the same `ECFLOW_VERSION` and `ECFLOW_PACKAGE` build arguments described in the - `ecflow-server` section below, then pushes it to `eccr.ecmwf.int/ecflow-dev-environments/ecflow-serveronly-dev`. +2. The `dockerize` job, using the same matrix, downloads the matching artefact into the matching + `ecflow-server/` or `ecflow-all/` directory and builds the Docker image from that directory's + `Dockerfile`, then pushes it to `eccr.ecmwf.int/ecflow-dev-environments/`. ## Building the images @@ -70,13 +71,13 @@ Build the image, passing the package version and filename as build arguments: ```bash docker build \ - --build-arg ECFLOW_VERSION=_ \ --build-arg ECFLOW_PACKAGE=ecflow-_-Linux_x86_64.deb \ -t ecflow-serveronly-dev:latest \ ecflow-server/ ``` -Note: provide `.deb` filename explicitly to match the file produced in Step 1. To avoid defining version and package, +Note: provide the `.deb` filename explicitly to match the file produced in Step 1. To avoid defining version and +package, rename the package to the default name `ecflow-latest-Linux.deb`. The ecflow ports are configurable via the `ECFLOW_SERVER_PORT` (default `8888`) and `ECFLOW_REST_PORT` (default `8889`) @@ -87,3 +88,76 @@ Run the image with, for example: ```bash docker run --rm -p 8888:8888 -p 8889:8889 ecflow-serveronly-dev:latest ``` + +### Building the `ecflow-all` image + +This section describes the process to build a Docker image capable of running `ecflow_ui`, connected to a remote ecFlow +server (for example on ECMWF's HPC) through a `proxychains4` + SSH SOCKS tunnel opened on the Docker host. Although it +has `ecflow-server`, this image is designed to run `ecflow_ui` only, and does not start a server itself. + +#### Step 1 (repeated): Build the ecFlow Debian package + +Follow Step 1 above to produce a `.deb`. In this case, because the package must include the ecflow UI, use preset +`linux.gcc.all.relwithdebinfo` for the configuration, for example: + +```bash +./ecflow-server.build.package.sh --preset linux.gcc.all.relwithdebinfo --output_dir ecflow-all +``` + +Ensure this the `.deb` is made available in `ecflow-all/` to support building the Docker image in Step 3. +The `--output_dir` option above copies the package into that directory automatically. + +#### Step 3: Build the `ecflow-all` container image + +```bash +docker build \ + --build-arg ECFLOW_PACKAGE=ecflow-_-Linux_x86_64.deb \ + -t ecflow-all-dev:latest \ + ecflow-all/ +``` + +Note: provide the `.deb` filename explicitly to match the file produced in Step 1, or rename the +package to the default name `ecflow-latest-Linux.deb` to avoid defining version and package. + +#### Running `ecflow_ui` + +The `ecflow_ui` can easily be launched using the `docker_ecflow_ui.sh` script. + +This utility script handles all the nitty-gritty details of using Docker to run `ecflow_ui`: + + - it forwards X11 to the host, + - mounts and passes `-confd` for the ecflow_ui configuration directory + - applies `xhost +`. + +It is self-contained, so it can be run either as a local checkout or fetched and piped straight into `bash`: + +```bash +./docker_ecflow_ui.sh +# or +curl -fsSL /docker_ecflow_ui.sh | bash -s -- --proxychains=on +``` + +Options (`./docker_ecflow_ui.sh --help` for the full list): + +| Option | Default | Purpose | +|-------------------------|-----------------------------------------------------------------------------|------------------------------------------------------------------------------------------| +| `-c`, `--cfg DIR` | `~/.ecflow_ui_v5` | ecflow_ui configuration directory (bind-mounted, passed via `-confd`) | +| `-i`, `--image IMAGE` | `ecflow-all-dev:latest` | Docker image to run | +| `--proxychains=on\|off` | `off` | Route connections through proxychains4 + the SSH SOCKS tunnel | +| `--authtokens FILE` | unset (falls back to `ECF_AUTHTOKENS` from the calling environment, if set) | Auth tokens file, bind-mounted read-only and exposed to `ecflow_ui` via `ECF_AUTHTOKENS` | + +By default (`--proxychains=off`) the script bypasses the image's own `ENTRYPOINT`(`launch-ecflow_ui.sh`) and runs +`ecflow_ui` directly — appropriate when the target ecFlow server is already reachable without a tunnel. + +With `--proxychains=on`, the image's `ENTRYPOINT` resolves the SOCKS proxy host (`host.docker.internal` by default) to +a literal IP address at container start, generates `/etc/proxychains4.conf` from it, and execs `ecflow_ui` under +`proxychains4`. Two things are required on the host side in that case: + +1. An SSH SOCKS tunnel to the HPC environment, bound to more than loopback so the container can reach it + through `host.docker.internal`: + + ```bash + ssh -g -D 9050 user@hpc-gateway + ``` + +2. On macOS, an X server (e.g. XQuartz) running (`docker_ecflow_ui.sh` applies `xhost +` itself). diff --git a/releng/dockit/docker_ecflow_ui.sh b/releng/dockit/docker_ecflow_ui.sh new file mode 100755 index 000000000..9f36d06cd --- /dev/null +++ b/releng/dockit/docker_ecflow_ui.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# +# docker_ecflow_ui.sh - launch ecflow_ui in a Docker container (ecflow-all-dev:latest by default), +# with its X11 display forwarded to the host and, optionally, its connections routed through +# proxychains4 + an SSH SOCKS tunnel opened on the host (see ecflow-all/launch-ecflow_ui.sh). +# +# Designed to be run either as a local checkout, or fetched and piped straight into bash: +# +# curl -fsSL /docker_ecflow_ui.sh | bash -s -- --proxychains=on +# +# (arguments go after the `--`, per bash's own `-s` convention; with no `--`, no arguments can be +# passed and every default below applies.) + +set -e +set -u +set -o pipefail + +CFG_DIR="${HOME}/.ecflow_ui_v5" +IMAGE="eccr.ecmwf.int/ecflow-dev-environments/ecflow-all-dev:latest" +PLATFORM="linux/amd64" +PROXYCHAINS="off" +# An ECF_AUTHTOKENS already set in the environment this script runs in is equivalent to passing +# --authtokens with that value; the flag, if also given, takes precedence over it. +AUTHTOKENS="${ECF_AUTHTOKENS:-}" + +usage() { + cat <<'EOF' +Usage: docker_ecflow_ui.sh [options] + +Launch ecflow_ui in a Docker container, with its X11 display forwarded to the host and, +optionally, its connections routed through a proxychains4 + SSH SOCKS tunnel. + +Options: + -c, --cfg DIR ecflow_ui configuration directory (default: ~/.ecflow_ui_v5) + -i, --image IMAGE Docker image to run (default: eccr.ecmwf.int/ecflow-dev-environments/ecflow-all-dev:latest) + --platform PLATFORM Docker platform to use (default: linux/amd64) + --proxychains=on|off + Route connections through proxychains4 + the SSH SOCKS tunnel baked + into the image's entrypoint (default: off) + --authtokens FILE Auth tokens file to mount into the container and expose to ecflow_ui + via ECF_AUTHTOKENS (default: unset). An ECF_AUTHTOKENS already set in + the calling environment is used as the default for this option, so + having it exported is equivalent to passing --authtokens explicitly; + an explicit --authtokens overrides it. + -h, --help Show this help and exit + +Prerequisites (macOS): + - An X server (e.g. XQuartz) running, with 'xhost +' applied (this script also applies it). + - If --proxychains=on, an SSH SOCKS tunnel bound to more than loopback, left open in another + terminal: + ssh -g -D 9050 user@hpc-gateway + +Examples: + ./docker_ecflow_ui.sh + ./docker_ecflow_ui.sh --proxychains=on + ./docker_ecflow_ui.sh -c ~/cfg/ecflow_ui -i ecflow-all-dev:2026.1 + ./docker_ecflow_ui.sh --authtokens ~/.ecflow_server.tokens + ECF_AUTHTOKENS=~/.ecflow_server.tokens ./docker_ecflow_ui.sh + curl -fsSL /docker_ecflow_ui.sh | bash -s -- --proxychains=on +EOF +} + +while [ $# -gt 0 ]; do + case "$1" in + -c|--cfg) + CFG_DIR="$2" + shift 2 + ;; + --cfg=*) + CFG_DIR="${1#*=}" + shift + ;; + -i|--image) + IMAGE="$2" + shift 2 + ;; + --image=*) + IMAGE="${1#*=}" + shift + ;; + --platform) + PLATFORM="$2" + shift 2 + ;; + --proxychains) + PROXYCHAINS="$2" + shift 2 + ;; + --proxychains=*) + PROXYCHAINS="${1#*=}" + shift + ;; + --authtokens) + AUTHTOKENS="$2" + shift 2 + ;; + --authtokens=*) + AUTHTOKENS="${1#*=}" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "docker_ecflow_ui.sh: unknown option '$1'" >&2 + usage >&2 + exit 1 + ;; + esac +done + +case "${PROXYCHAINS}" in + on|off) ;; + *) + echo "docker_ecflow_ui.sh: --proxychains must be 'on' or 'off', got '${PROXYCHAINS}'" >&2 + exit 1 + ;; +esac + +if ! command -v docker >/dev/null 2>&1; then + echo "docker_ecflow_ui.sh: docker is not installed or not on PATH" >&2 + exit 1 +fi + +mkdir -p "${CFG_DIR}" +CFG_DIR=$(cd "${CFG_DIR}" && pwd) + +if command -v xhost >/dev/null 2>&1; then + xhost + >/dev/null +else + echo "docker_ecflow_ui.sh: warning: 'xhost' not found; skipping X11 access control setup" >&2 +fi + +# proxychains=off bypasses the image's own ENTRYPOINT (launch-ecflow_ui.sh, which always sets up +# proxychains4) and runs ecflow_ui directly instead. +# Built as a single array, always with at least one element, rather than splicing in a separate +# (possibly empty) array of --entrypoint args: bash 3.2 (macOS's default /bin/bash) raises +# "unbound variable" under `set -u` when expanding "${arr[@]}" on a genuinely empty array. +DOCKER_ARGS=( + --rm + -P + -e DISPLAY=host.docker.internal:0 + -v /tmp/.X11-unix:/tmp/.X11-unix + -v "${CFG_DIR}:${CFG_DIR}" +) +if [ "${PROXYCHAINS}" = "off" ]; then + DOCKER_ARGS+=(--entrypoint /usr/local/bin/ecflow_ui) +fi + +if [ -n "${AUTHTOKENS}" ]; then + if [ ! -f "${AUTHTOKENS}" ]; then + echo "docker_ecflow_ui.sh: --authtokens file '${AUTHTOKENS}' does not exist" >&2 + exit 1 + fi + AUTHTOKENS_DIR=$(cd "$(dirname "${AUTHTOKENS}")" && pwd) + AUTHTOKENS="${AUTHTOKENS_DIR}/$(basename "${AUTHTOKENS}")" + # Mounted read-only, at the same path inside the container as on the host, so ECF_AUTHTOKENS + # can be set to that one path without needing to translate it. + DOCKER_ARGS+=(-v "${AUTHTOKENS}:${AUTHTOKENS}:ro" -e "ECF_AUTHTOKENS=${AUTHTOKENS}") +fi + +# No -i/-t: ecflow_ui is a GUI app driven over X11, not an interactive TTY program, and dropping +# them keeps this working when stdin is not a terminal - as when this script is itself piped into +# bash via curl. docker run still streams stdout/stderr to this terminal and blocks until exit. +exec docker run "${DOCKER_ARGS[@]}" --platform "${PLATFORM}" "${IMAGE}" -confd "${CFG_DIR}" diff --git a/releng/dockit/ecflow-all/Dockerfile b/releng/dockit/ecflow-all/Dockerfile new file mode 100644 index 000000000..064502ef1 --- /dev/null +++ b/releng/dockit/ecflow-all/Dockerfile @@ -0,0 +1,37 @@ + +FROM debian:13.5-slim + +RUN apt-get update \ + && apt-get install -y \ + python3 \ + python3-venv \ + libssl-dev \ + libboost-all-dev \ + qt6-base-dev \ + qt6-svg-dev \ + qt6-charts-dev \ + qt6-5compat-dev \ + proxychains4 \ + && rm -rf /var/lib/apt/lists/* + +# Provision ecFlow dependencies +RUN python3 -m venv /opt/local/python \ + && /opt/local/python/bin/pip install "troika>=0.2.6" + +# Provision ecFlow package +## This Dockerfile assumes that the ecFlow package is available in the build context i.e. .../imachination/ecflow. +# If you are using a different version, change the ECFLOW_PACKAGE variable accordingly. +ARG ECFLOW_VERSION="latest" +ARG ECFLOW_PACKAGE="ecflow-${ECFLOW_VERSION}-Linux.deb" +COPY ${ECFLOW_PACKAGE} /tmp/${ECFLOW_PACKAGE} +RUN dpkg -i /tmp/${ECFLOW_PACKAGE} \ + && rm /tmp/${ECFLOW_PACKAGE} + +ARG ECFLOW_UI_LAUNCH="launch-ecflow_ui.sh" +COPY ${ECFLOW_UI_LAUNCH} /opt/local/bin/${ECFLOW_UI_LAUNCH} +RUN chmod +x /opt/local/bin/${ECFLOW_UI_LAUNCH} + +# Set the working directory +WORKDIR /workspace + +ENTRYPOINT ["/opt/local/bin/launch-ecflow_ui.sh"] diff --git a/releng/dockit/ecflow-all/launch-ecflow_ui.sh b/releng/dockit/ecflow-all/launch-ecflow_ui.sh new file mode 100755 index 000000000..64ff52c4b --- /dev/null +++ b/releng/dockit/ecflow-all/launch-ecflow_ui.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -e +set -u +set -o pipefail + +# Resolve the SOCKS5 proxy host (the SSH tunnel opened on the Docker host) to a literal IP address. +# proxychains-ng refuses a non-numeric address for the first proxy in a chain, since that hop is +# dialled directly, before any tunnel exists through which a hostname could be resolved remotely. +# The resolved IP is therefore generated at container start, not baked into the image. + +PROXY_HOST=${PROXY_HOST:-host.docker.internal} +PROXY_PORT=${PROXY_PORT:-9050} + +PROXY_HOST_IP=$(getent hosts "${PROXY_HOST}" | awk '{ print $1; exit }') +if [ -z "${PROXY_HOST_IP}" ]; then + echo "launch-ecflow_ui.sh: could not resolve ${PROXY_HOST} to an IP address" >&2 + exit 1 +fi + +# Under LD_PRELOAD, proxychains4 intercepts every outbound connect() in the process, including +# traffic that has nothing to do with the HPC connection - notably the X11 connection back to +# host.docker.internal:0 for the Qt UI. host.docker.internal resolves to the same host/subnet as +# the proxy itself, so without this exclusion that local traffic gets shoved through the SOCKS +# tunnel too and times out. localnet routes matching traffic direct, bypassing the chain. +PROXY_HOST_SUBNET=$(echo "${PROXY_HOST_IP}" | awk -F. '{ print $1"."$2"."$3".0" }') + +cat > /etc/proxychains4.conf <