Add SOS-backed macOS OpenSHMEM packages

Author: allnes

.github/workflows/nightly-openmpi.yml

@@ -22,11 +22,13 @@ jobs:
           - platform: linux-x86_64
             runner: ubuntu-24.04
             shmem: required
+            sos: false
             shmem_validation: passed
           - platform: macos-arm64
             runner: macos-15
             shmem: disabled
-            shmem_validation: unsupported
+            sos: true
+            shmem_validation: passed
     env:
       CCACHE_DIR: ${{ github.workspace }}/_ccache
       MPI_EXTENSIONS_CCACHE: on
@@ -41,7 +43,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: _ccache
-          key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.platform }}-${{ hashFiles('manifests/openmpi-version.json', '.gitmodules') }}
+          key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.platform }}-${{ hashFiles('manifests/openmpi-version.json', 'manifests/sos-version.json', '.gitmodules') }}
           restore-keys: |
             ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.platform }}-
             ccache-${{ runner.os }}-${{ runner.arch }}-
@@ -73,7 +75,7 @@ jobs:
         run: |
           set -euo pipefail
           brew update
-          brew install autoconf automake bison ccache flex libtool pkg-config rsync
+          brew install autoconf automake bison ccache flex libfabric libtool pkg-config rsync
           {
             echo "$(brew --prefix bison)/bin"
             echo "$(brew --prefix flex)/bin"
@@ -92,30 +94,32 @@ jobs:
         run: |
           set -euo pipefail
           ccache -z
-          ./scripts/build_openmpi.sh \
+          ./scripts/build_openmpi.py \
             --prefix "$PWD/_install/openmpi" \
             --build-dir "$PWD/_build/openmpi-${{ matrix.platform }}" \
             --enable-shmem "${{ matrix.shmem }}" \
             --ccache on
 
+      - name: Build Sandia OpenSHMEM/SOS
+        if: matrix.sos
+        run: |
+          set -euo pipefail
+          ./scripts/build_sos_shmem.py \
+            --prefix "$PWD/_install/openmpi" \
+            --build-dir "$PWD/_build/sos-${{ matrix.platform }}" \
+            --mpi-prefix "$PWD/_install/openmpi" \
+            --ccache on
+
       - name: Validate MPI
-        run: ./scripts/smoke_test_mpi.sh "$PWD/_install/openmpi"
+        run: ./scripts/smoke_test_mpi.py "$PWD/_install/openmpi"
 
       - name: Validate OpenSHMEM
-        if: matrix.shmem == 'required'
-        run: ./scripts/smoke_test_shmem.sh "$PWD/_install/openmpi"
-
-      - name: Record unsupported OpenSHMEM on macOS
-        if: matrix.shmem != 'required'
-        run: |
-          set -euo pipefail
-          MPI_EXTENSIONS_ALLOW_UNSUPPORTED_SHMEM=1 \
-            ./scripts/smoke_test_shmem.sh "$PWD/_install/openmpi"
+        run: ./scripts/smoke_test_shmem.py "$PWD/_install/openmpi"
 
       - name: Package Open MPI
         run: |
           set -euo pipefail
-          ./scripts/package_openmpi.sh \
+          ./scripts/package_openmpi.py \
             --prefix "$PWD/_install/openmpi" \
             --out "$PWD/_dist" \
             --platform "${{ matrix.platform }}" \

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "external/ompi"]
 	path = external/ompi
 	url = https://github.com/open-mpi/ompi
+[submodule "external/sos"]
+	path = external/sos
+	url = https://github.com/Sandia-OpenSHMEM/SOS.git

README.md

@@ -2,29 +2,38 @@
 
 `mpi-extensions` supplies Open MPI binary packages for
 [`parallel_programming_course`](https://github.com/learning-process/parallel_programming_course).
-The packages are built from the Open MPI git submodule at `external/ompi` and
-are intended for rootless CI use: download, extract, add `bin/` to `PATH`, and
-let CMake discover `mpicc` and `mpicxx`.
+The packages are built from pinned git submodules and are intended for rootless
+CI use: download, extract, and pass the unpacked package prefix to the course
+CMake configure step.
 
 This repository publishes only one public binary channel: the moving `nightly`
 pre-release named `Nightly Open MPI Extensions`. Stable versioned releases are
 not supported here.
 
-## Open MPI pin
+## Source Pins
 
-Open MPI is included as a git submodule, not vendored source:
+Open MPI and Sandia OpenSHMEM/SOS are included as git submodules, not vendored
+source:
 
 - URL: `https://github.com/open-mpi/ompi`
 - Path: `external/ompi`
 - Ref: `v5.0.10`
 - Commit: `0d48030d410ae8f56790933135b28be1b3920ba1`
 
-The same data is recorded in `manifests/openmpi-version.json`.
+- URL: `https://github.com/Sandia-OpenSHMEM/SOS.git`
+- Path: `external/sos`
+- Ref: `v1.5.3`
+- Commit: `45e6e7eb1a9b099a418237a9e677eb6603222c84`
 
-Linux packages are configured with MPI and OpenSHMEM/OSHMEM enabled and bundle
-the UCX runtime libraries used by OpenSHMEM. Open MPI 5.0.x marks OSHMEM as
-Linux-only in its configure logic, so macOS packages are MPI-only and their
-package manifests record OpenSHMEM validation as `unsupported`.
+The same data is recorded in `manifests/openmpi-version.json` and
+`manifests/sos-version.json`.
+
+Linux packages are configured with MPI and Open MPI OSHMEM/OpenSHMEM enabled
+and bundle the UCX runtime libraries used by OpenSHMEM. Open MPI 5.0.x marks
+OSHMEM as Linux-only in its configure logic, so macOS packages use Open MPI for
+MPI and Sandia OpenSHMEM/SOS for SHMEM. SOS is built against the package's Open
+MPI PMIx/hwloc install and libfabric/OFI, then packaged with the needed runtime
+libraries for rootless CI use.
 
 ## Nightly assets
 
@@ -46,22 +55,22 @@ Archives unpack under a single top-level directory and contain `bin/`, `lib/`,
 For local use from this repository:
 
 ```bash
-./scripts/install_from_nightly_release.sh \
+./scripts/install_from_nightly_release.py \
   --repo learning-process/mpi-extensions \
   --platform auto \
   --prefix "$PWD/_deps/mpi-extensions-openmpi"
 ```
 
 The installer downloads only the `nightly` pre-release, verifies the SHA256
-checksum, extracts into the requested prefix, and prints the environment values
-needed by downstream builds.
+checksum, extracts into the requested prefix, and prints environment values for
+manual debugging.
 
 From another repository, such as `parallel_programming_course`, fetch the
 installer script directly from this repository:
 
 ```bash
 curl -fsSL \
-  https://raw.githubusercontent.com/learning-process/mpi-extensions/main/scripts/install_from_nightly_release.sh \
+  https://raw.githubusercontent.com/learning-process/mpi-extensions/main/scripts/install_from_nightly_release.py \
   -o /tmp/install_mpi_extensions.py
 python3 /tmp/install_mpi_extensions.py \
   --repo learning-process/mpi-extensions \
@@ -72,29 +81,26 @@ python3 /tmp/install_mpi_extensions.py \
 Recommended `parallel_programming_course` CI setup:
 
 ```bash
-export MPI_EXTENSIONS_HOME="$PWD/_deps/mpi-extensions-openmpi"
-export MPI_HOME="$MPI_EXTENSIONS_HOME"
-export OPAL_PREFIX="$MPI_EXTENSIONS_HOME"
-export PATH="$MPI_EXTENSIONS_HOME/bin:$PATH"
-export LD_LIBRARY_PATH="$MPI_EXTENSIONS_HOME/lib:${LD_LIBRARY_PATH:-}"
-export DYLD_LIBRARY_PATH="$MPI_EXTENSIONS_HOME/lib:${DYLD_LIBRARY_PATH:-}"
-
-cmake -S . -B build
+MPI_EXTENSIONS_HOME="$PWD/_deps/mpi-extensions-openmpi"
+cmake -S . -B build -D PPC_MPI_EXTENSIONS_HOME="$MPI_EXTENSIONS_HOME"
 cmake --build build --parallel
 ```
 
-Do not install Homebrew or apt Open MPI in the course workflow. The binary
-package is meant to be self-contained for CMake discovery through the wrapper
-compilers.
+Do not install Homebrew or apt Open MPI in the course workflow. The course CMake
+configuration must receive `PPC_MPI_EXTENSIONS_HOME`; it then points MPI
+discovery and runtime setup at this package.
 
 ## Local build
 
+All repository helper scripts in `scripts/` are Python executables with `.py`
+extensions.
+
 Install the normal compiler/autotools dependencies first. On Ubuntu this
 includes `build-essential`, `autoconf`, `automake`, `libtool`, `flex`, `bison`,
 `pkg-config`, `ccache`, `zlib1g-dev`, and `libucx-dev`; `libucx-dev` is required
 for the Linux OpenSHMEM provider. On macOS the workflow uses Homebrew packages
-for `autoconf`, `automake`, `bison`, `ccache`, `flex`, `libtool`, `pkg-config`,
-and `rsync`.
+for `autoconf`, `automake`, `bison`, `ccache`, `flex`, `libfabric`, `libtool`,
+`pkg-config`, and `rsync`.
 
 Initialize submodules first:
 
@@ -111,14 +117,14 @@ Build and validate on Linux:
 ```bash
 export CCACHE_DIR="$MPI_EXTENSIONS_WORK_ROOT/ccache"
 
-./scripts/build_openmpi.sh \
+./scripts/build_openmpi.py \
   --prefix "$MPI_EXTENSIONS_WORK_ROOT/install/openmpi" \
   --build-dir "$MPI_EXTENSIONS_WORK_ROOT/build/openmpi-linux" \
   --enable-shmem required \
   --ccache on
-./scripts/smoke_test_mpi.sh "$MPI_EXTENSIONS_WORK_ROOT/install/openmpi"
-./scripts/smoke_test_shmem.sh "$MPI_EXTENSIONS_WORK_ROOT/install/openmpi"
-./scripts/package_openmpi.sh \
+./scripts/smoke_test_mpi.py "$MPI_EXTENSIONS_WORK_ROOT/install/openmpi"
+./scripts/smoke_test_shmem.py "$MPI_EXTENSIONS_WORK_ROOT/install/openmpi"
+./scripts/package_openmpi.py \
   --prefix "$MPI_EXTENSIONS_WORK_ROOT/install/openmpi" \
   --out "$MPI_EXTENSIONS_WORK_ROOT/build/dist"
 ```
@@ -128,24 +134,28 @@ Build and validate on macOS:
 ```bash
 export CCACHE_DIR="$MPI_EXTENSIONS_WORK_ROOT/ccache"
 
-./scripts/build_openmpi.sh \
+./scripts/build_openmpi.py \
   --prefix "$MPI_EXTENSIONS_WORK_ROOT/install/openmpi" \
   --build-dir "$MPI_EXTENSIONS_WORK_ROOT/build/openmpi-macos" \
   --enable-shmem disabled \
   --ccache on
-./scripts/smoke_test_mpi.sh "$MPI_EXTENSIONS_WORK_ROOT/install/openmpi"
-MPI_EXTENSIONS_ALLOW_UNSUPPORTED_SHMEM=1 \
-  ./scripts/smoke_test_shmem.sh "$MPI_EXTENSIONS_WORK_ROOT/install/openmpi"
-./scripts/package_openmpi.sh \
+./scripts/build_sos_shmem.py \
+  --prefix "$MPI_EXTENSIONS_WORK_ROOT/install/openmpi" \
+  --build-dir "$MPI_EXTENSIONS_WORK_ROOT/build/sos-macos" \
+  --mpi-prefix "$MPI_EXTENSIONS_WORK_ROOT/install/openmpi" \
+  --ccache on
+./scripts/smoke_test_mpi.py "$MPI_EXTENSIONS_WORK_ROOT/install/openmpi"
+./scripts/smoke_test_shmem.py "$MPI_EXTENSIONS_WORK_ROOT/install/openmpi"
+./scripts/package_openmpi.py \
   --prefix "$MPI_EXTENSIONS_WORK_ROOT/install/openmpi" \
   --out "$MPI_EXTENSIONS_WORK_ROOT/build/dist" \
   --platform macos-arm64 \
-  --shmem-smoke unsupported
+  --shmem-smoke passed
 ```
 
-## Updating Open MPI
+## Updating Pins
 
-To move the pin:
+To move the Open MPI pin:
 
 ```bash
 git -C external/ompi fetch --tags origin
@@ -154,8 +164,17 @@ git -C external/ompi submodule update --init --recursive
 git add external/ompi manifests/openmpi-version.json README.md
 ```
 
-Update `manifests/openmpi-version.json` and this README with the exact ref and
-commit. Use a stable tag or explicit commit, never a floating branch.
+To move the SOS pin:
+
+```bash
+git -C external/sos fetch --tags origin
+git -C external/sos checkout <stable-tag-or-commit>
+git -C external/sos submodule update --init --recursive
+git add external/sos manifests/sos-version.json README.md
+```
+
+Update the matching manifest and this README with the exact ref and commit. Use
+a stable tag or explicit commit, never a floating branch.
 
 ## Workflow
 

docs/parallel_programming_course_integration.md

@@ -9,37 +9,33 @@ Install the current nightly package before configuring the course build:
 
 ```bash
 curl -fsSL \
-  https://raw.githubusercontent.com/learning-process/mpi-extensions/main/scripts/install_from_nightly_release.sh \
+  https://raw.githubusercontent.com/learning-process/mpi-extensions/main/scripts/install_from_nightly_release.py \
   -o /tmp/install_mpi_extensions.py
 python3 /tmp/install_mpi_extensions.py \
   --repo learning-process/mpi-extensions \
   --platform auto \
   --prefix "$PWD/_deps/mpi-extensions-openmpi"
 
-export MPI_EXTENSIONS_HOME="$PWD/_deps/mpi-extensions-openmpi"
-export MPI_HOME="$MPI_EXTENSIONS_HOME"
-export OPAL_PREFIX="$MPI_EXTENSIONS_HOME"
-export PATH="$MPI_EXTENSIONS_HOME/bin:$PATH"
-export LD_LIBRARY_PATH="$MPI_EXTENSIONS_HOME/lib:${LD_LIBRARY_PATH:-}"
-export DYLD_LIBRARY_PATH="$MPI_EXTENSIONS_HOME/lib:${DYLD_LIBRARY_PATH:-}"
-
-cmake -S . -B build
+MPI_EXTENSIONS_HOME="$PWD/_deps/mpi-extensions-openmpi"
+cmake -S . -B build -D PPC_MPI_EXTENSIONS_HOME="$MPI_EXTENSIONS_HOME"
 cmake --build build --parallel
 ```
 
-The package is intended to be discovered by CMake through the Open MPI wrapper
-compilers in `bin/`. Keep `PATH` pointed at the package before calling CMake so
-`find_package(MPI)` resolves `mpicc` and `mpicxx` from this archive.
+The package is intended to be passed to course CMake through
+`PPC_MPI_EXTENSIONS_HOME`. The course configuration then points MPI discovery at
+the wrapper compilers in the package and writes the runtime environment needed
+by `scripts/run_tests.py`.
 
 ## Platform behavior
 
-Linux packages are built with MPI and OpenSHMEM/OSHMEM enabled, bundle the UCX
-runtime libraries needed by OpenSHMEM, and are validated with two-rank C smoke
-tests.
+Linux packages are built with MPI and Open MPI OSHMEM/OpenSHMEM enabled,
+bundle the UCX runtime libraries needed by OpenSHMEM, and are validated with
+two-rank C smoke tests.
 
 Open MPI 5.0.x marks OSHMEM as Linux-only in its configure logic, so macOS
-packages provide MPI tooling only. Their manifest records the OpenSHMEM smoke
-test as `unsupported`.
+packages provide MPI through Open MPI and SHMEM through Sandia OpenSHMEM/SOS
+built against the same Open MPI PMIx/hwloc prefix and libfabric/OFI. macOS
+packages are also validated with two-rank MPI and SHMEM C smoke tests.
 
 ## Nightly channel
 

external/sos

@@ -3,5 +3,5 @@
   "openmpi_ref": "v5.0.10",
   "openmpi_commit": "0d48030d410ae8f56790933135b28be1b3920ba1",
   "submodule_path": "external/ompi",
-  "notes": "Pinned as a git submodule. Open MPI OSHMEM/OpenSHMEM support is enabled and validated on Linux; upstream Open MPI 5.0.x marks OSHMEM as Linux-only."
+  "notes": "Pinned as a git submodule. Open MPI provides MPI on all supported platforms and Open MPI OSHMEM/OpenSHMEM on Linux. macOS OpenSHMEM support is supplied by the pinned Sandia OpenSHMEM/SOS submodule over Open MPI PMIx and libfabric/OFI."
 }

manifests/openmpi-version.json

@@ -0,0 +1,7 @@
+{
+  "sos_git_url": "https://github.com/Sandia-OpenSHMEM/SOS.git",
+  "sos_ref": "v1.5.3",
+  "sos_commit": "45e6e7eb1a9b099a418237a9e677eb6603222c84",
+  "submodule_path": "external/sos",
+  "notes": "Pinned as a git submodule. Used to provide macOS OpenSHMEM/SHMEM support over Open MPI PMIx and libfabric/OFI because Open MPI 5.0.x OSHMEM configure support is Linux-only."
+}