Skip to content

Improve --bwrap to work with r/o filesystems (overlayfs) and --subdir-{modules,software} #5173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
smoors merged 6 commits into from
Jun 29, 2026
+60 −8
Merged

Improve --bwrap to work with r/o filesystems (overlayfs) and --subdir-{modules,software} #5173

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
bc88860
Use subdir_software/subdir_modules for --bwrap
bartoldeman Apr 13, 2026
4d8e6a4
Use overlayfs with bwrap if we cannot create installdir
bartoldeman Apr 13, 2026
cb42ab5
Merge branch 'develop' into pr5173
Jun 13, 2026
d71ac0d
add end-to-end test for bwrap with overlay on ubuntu 26.06
Jun 13, 2026
3c376be
remove --try-amend
Jun 14, 2026
cd0267c
add bwrap_installpath_software and bwrap_installpath_modules to the j…
Jun 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 33 additions & 2 deletions .github/workflows/end2end_bwrap.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on: [push, pull_request]
jobs:
end2end:
name: End-to-end test (runner VM)
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04

steps:
- name: Check out the repo
Expand All @@ -14,7 +14,7 @@ jobs:
- name: Install system packages
run: sudo apt-get install -y lmod bubblewrap

- name: Allow unprivileged user namespaces (Ubuntu 24.04)
- name: Allow unprivileged user namespaces (Ubuntu 26.04)
run: |
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0
Expand Down Expand Up @@ -77,3 +77,34 @@ jobs:

grep 'no UnZip module dir found: OK' /tmp/out-UnZip-mod && grep 'no bzip2 module dir found: OK' /tmp/out-bzip2-mod && grep 'UnZip software dir in .* is empty' /tmp/out-UnZip-sw-dir && grep 'bzip2 software dir in .* is empty' /tmp/out-bzip2-sw-dir && ls -l /tmp/bwrap-installpath/modules/all/{bzip2/1.0.8.lua,UnZip/6.0.lua} && ls -l /tmp/bwrap-installpath/software/{UnZip,bzip2}/*/bin/* && ls -l /tmp/bwrap-installpath/software/{UnZip,bzip2}/*/easybuild/*log*
EOF

- name: End-to-end test of installing UnZip with EasyBuild in bwrap namespace with overlay
shell: bash
run: |
bash -l <<EOF
cat /etc/os-release
# bwrap version needs to be >=0.10 for --overlay and --overlay-src support
bwrap --version

source /tmp/eb_env

# remove previously created install directories
rm -r $HOME/.local/easybuild/software/* /tmp/bwrap-installpath

# make install directory read-only to force using overlay
chmod -w $HOME/.local/easybuild/software

# test installation of UnZip + bzip2 dependency
eb UnZip-6.0.eb --trace --robot --experimental --bwrap --bwrap-installpath /tmp/bwrap-installpath --filter-deps=binutils

# no module files should be created in configured install path (default: $HOME/.local/easybuild),
# not for UnZip, nor for bzip2 dependency
(test -d $HOME/.local/easybuild/modules/all/UnZip || echo 'no UnZip module dir found: OK') > /tmp/out-UnZip-mod
(test -d $HOME/.local/easybuild/modules/all/bzip2 || echo 'no bzip2 module dir found: OK') > /tmp/out-bzip2-mod

# software subdirectories in configured install path should be empty
(test -e $HOME/.local/easybuild/software/UnZip/6.0/* || echo 'UnZip software dir in $HOME/.local/easybuild is empty: OK') > /tmp/out-UnZip-sw-dir
(test -e $HOME/.local/easybuild/software/bzip2/1.0.8/* || echo 'bzip2 software dir in $HOME/.local/easybuild is empty: OK') > /tmp/out-bzip2-sw-dir

grep 'no UnZip module dir found: OK' /tmp/out-UnZip-mod && grep 'no bzip2 module dir found: OK' /tmp/out-bzip2-mod && grep 'UnZip software dir in .* is empty' /tmp/out-UnZip-sw-dir && grep 'bzip2 software dir in .* is empty' /tmp/out-bzip2-sw-dir && ls -l /tmp/bwrap-installpath/modules/all/{bzip2/1.0.8.lua,UnZip/6.0.lua} && ls -l /tmp/bwrap-installpath/software/{UnZip,bzip2}/*/bin/* && ls -l /tmp/bwrap-installpath/software/{UnZip,bzip2}/*/easybuild/*log*
EOF
33 changes: 27 additions & 6 deletions easybuild/tools/bwrap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@

from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.config import install_path
from easybuild.tools.config import install_path, ConfigurationVariables
from easybuild.tools.filetools import mkdir, write_file
from easybuild.tools.utilities import trace_msg

Expand All @@ -47,6 +47,8 @@
'bwrap_cmd': [],
'bwrap_eb_options': [],
'bwrap_installpath': '',
'bwrap_installpath_software': '',
'bwrap_installpath_modules': '',
'installpath_modules': '',
'installpath_software': '',
'modules_to_install': set(),
Expand Down Expand Up @@ -107,23 +109,42 @@ def prepare_bwrap(bwrap_installpath):

set_bwrap_info('installpath_modules', install_path(typ='modules'))

bwrap_modules_installpath = os.path.join(bwrap_installpath, 'modules')
variables = ConfigurationVariables()
bwrap_installpath_software = os.path.join(bwrap_installpath, variables['subdir_software'])
bwrap_installpath_modules = os.path.join(bwrap_installpath, variables['subdir_modules'])
set_bwrap_info('bwrap_installpath_software', bwrap_installpath_software)
set_bwrap_info('bwrap_installpath_modules', bwrap_installpath_modules)

bwrap_cmd = ['bwrap', '--dev-bind', '/', '/']

# bind mount all software directories
for mod in sorted(get_bwrap_info('modules_to_install')):
installdir = os.path.join(os.path.realpath(installpath_software), mod)
bwrap_installdir = os.path.join(bwrap_installpath, 'software', mod)
mkdir(installdir, parents=True)
bwrap_installdir = os.path.join(bwrap_installpath_software, mod)
use_overlayfs = False
try:
mkdir(installdir, parents=True)
except EasyBuildError:
# if we can't create the external installation directory, try to use overlayfs
use_overlayfs = True
mkdir(bwrap_installdir, parents=True)
bwrap_cmd.extend(['--bind', bwrap_installdir, installdir])
if use_overlayfs:
bwrap_workdir = os.path.join(bwrap_installpath, 'workdir', mod)
mkdir(bwrap_workdir, parents=True)
# go up the tree until we find a directory that exists
while not os.path.exists(installdir):
installdir = os.path.dirname(installdir)
bwrap_installdir = os.path.dirname(bwrap_installdir)
opts = ['--overlay-src', installdir, '--overlay', bwrap_installdir, bwrap_workdir, installdir]
else:
opts = ['--bind', bwrap_installdir, installdir]
bwrap_cmd.extend(opts)

set_bwrap_info('bwrap_cmd', bwrap_cmd)
bwrap_cmd_str = ' '.join(bwrap_cmd)

# disable `--bwrap` to prepare for a real installation (in bwrap namespace)
bwrap_eb_options = ['--disable-bwrap', f'--installpath-modules={bwrap_modules_installpath}']
bwrap_eb_options = ['--disable-bwrap', f'--installpath-modules={bwrap_installpath_modules}']
set_bwrap_info('bwrap_eb_options', bwrap_eb_options)

_log.info(f'Info needed for bwrap: {_bwrap_info}')
Expand Down