Fix Python bindings version ignored in system tests

changelog-entries/745.md

@@ -0,0 +1 @@
+- Fixed the Python bindings version being ignored in the system tests, where tutorial `run.sh` scripts created their own virtual environment and reinstalled an unpinned `pyprecice` over the image's pinned `PYTHON_BINDINGS_REF`. The `nutils` and `micro-manager` images, which previously relied on those per-case environments to supply `nutils` (and `pyprecice`), now install the required packages directly. [#745](https://github.com/precice/tutorials/pull/745)

tools/tests/component-templates/fenics-adapter.yaml

@@ -5,13 +5,16 @@ build:
       - {{key}}={{value}}
     {% endfor %}
   target: fenics_adapter
+environment:
+  - PRECICE_TUTORIALS_NO_VENV=1
 depends_on:    
   prepare:
     condition: service_completed_successfully
 volumes:
   - {{ run_directory }}:/runs
 command: >
-  /bin/bash -c "id && 
+  /bin/bash -c "id &&
+  ([ -z \"$$PRECICE_TUTORIALS_NO_VENV\" ] || { python3 -c 'import precice' || { echo 'ERROR: PRECICE_TUTORIALS_NO_VENV set but precice not importable. Check Docker image.'; exit 1; }; }) &&
   cd '/runs/{{ tutorial_folder }}/{{ case_folder }}' &&
   {{ run }} &&
   rm -rf .venv __pycache__

tools/tests/component-templates/fenicsx-adapter.yaml

@@ -5,13 +5,16 @@ build:
       - {{key}}={{value}}
     {% endfor %}
   target: fenicsx_adapter
+environment:
+  - PRECICE_TUTORIALS_NO_VENV=1
 depends_on:    
   prepare:
     condition: service_completed_successfully
 volumes:
   - {{ run_directory }}:/runs
 command: >
-  /bin/bash -c "id && 
+  /bin/bash -c "id &&
+  ([ -z \"$$PRECICE_TUTORIALS_NO_VENV\" ] || { python3 -c 'import precice' || { echo 'ERROR: PRECICE_TUTORIALS_NO_VENV set but precice not importable. Check Docker image.'; exit 1; }; }) &&
   cd '/runs/{{ tutorial_folder }}/{{ case_folder }}' &&
   {{ run }} &&
   rm -rf .venv __pycache__

tools/tests/component-templates/micro-manager.yaml

@@ -5,13 +5,16 @@ build:
       - {{key}}={{value}}
     {% endfor %}
   target: micro_manager
+environment:
+  - PRECICE_TUTORIALS_NO_VENV=1
 depends_on:
   prepare:
     condition: service_completed_successfully
 volumes:
   - {{ run_directory }}:/runs
 command: >
   /bin/bash -c "id &&
+  ([ -z \"$$PRECICE_TUTORIALS_NO_VENV\" ] || { python3 -c 'import precice' || { echo 'ERROR: PRECICE_TUTORIALS_NO_VENV set but precice not importable. Check Docker image.'; exit 1; }; }) &&
   cd '/runs/{{ tutorial_folder }}/{{ case_folder }}' &&
   {{ run }} &&
   rm -rf .venv __pycache__

tools/tests/component-templates/nutils-adapter.yaml

@@ -5,13 +5,16 @@ build:
       - {{key}}={{value}}
     {% endfor %}
   target: nutils_adapter
+environment:
+  - PRECICE_TUTORIALS_NO_VENV=1
 depends_on:    
   prepare:
     condition: service_completed_successfully
 volumes:
   - {{ run_directory }}:/runs
 command: >
-  /bin/bash -c "id && 
+  /bin/bash -c "id &&
+  ([ -z \"$$PRECICE_TUTORIALS_NO_VENV\" ] || { python3 -c 'import precice' || { echo 'ERROR: PRECICE_TUTORIALS_NO_VENV set but precice not importable. Check Docker image.'; exit 1; }; }) &&
   cd '/runs/{{ tutorial_folder }}/{{ case_folder }}' &&
   {{ run }} &&
   rm -rf .venv __pycache__

tools/tests/component-templates/python-bindings.yaml

@@ -5,13 +5,16 @@ build:
       - {{key}}={{value}}
     {% endfor %}
   target: python_bindings
+environment:
+  - PRECICE_TUTORIALS_NO_VENV=1
 depends_on:    
   prepare:
     condition: service_completed_successfully
 volumes:
   - {{ run_directory }}:/runs
 command: >
-  /bin/bash -c "id && 
+  /bin/bash -c "id &&
+  ([ -z \"$$PRECICE_TUTORIALS_NO_VENV\" ] || { python3 -c 'import precice' || { echo 'ERROR: PRECICE_TUTORIALS_NO_VENV set but precice not importable. Check Docker image.'; exit 1; }; }) &&
   cd '/runs/{{ tutorial_folder }}/{{ case_folder }}' &&
   {{ run }} &&
   rm -rf .venv __pycache__

tools/tests/component-templates/su2-adapter.yaml

@@ -5,13 +5,16 @@ build:
       - {{key}}={{value}}
     {% endfor %}
   target: su2_adapter
+environment:
+  - PRECICE_TUTORIALS_NO_VENV=1
 depends_on:    
   prepare:
     condition: service_completed_successfully
 volumes:
   - {{ run_directory }}:/runs
 command: >
-  /bin/bash -c "id && 
+  /bin/bash -c "id &&
+  ([ -z \"$$PRECICE_TUTORIALS_NO_VENV\" ] || { python3 -c 'import precice' || { echo 'ERROR: PRECICE_TUTORIALS_NO_VENV set but precice not importable. Check Docker image.'; exit 1; }; }) &&
   cd '/runs/{{ tutorial_folder }}/{{ case_folder }}' &&
   SU2_RUN="/home/precice/SU2_RUN/bin" PYTHONPATH="/home/precice/SU2_RUN/bin:$PYTHONPATH" {{ run }} &&
   rm -rf .venv __pycache__

tools/tests/dockerfiles/ubuntu_2404/Dockerfile

@@ -116,25 +116,30 @@ USER precice
 WORKDIR /home/precice
 # Builds the precice python bindings for python3
 # Installs also matplotlib as its needed for the elastic-tube 1d fluid-python participant.
-RUN python3 -m venv /home/precice/venv && \
+RUN python3 -m venv --system-site-packages /home/precice/venv && \
     . /home/precice/venv/bin/activate && \
     pip3 install git+https://github.com/precice/python-bindings.git@${PYTHON_BINDINGS_REF} && \
-    pip3 install matplotlib 
+    pip3 install matplotlib
+ENV VIRTUAL_ENV=/home/precice/venv
+ENV PATH="/home/precice/venv/bin:$PATH"
 
-FROM precice_dependecies AS fenics_adapter
-COPY --from=python_bindings /home/precice/.local /home/precice/.local
+FROM python_bindings AS fenics_adapter
 USER root
 RUN add-apt-repository -y ppa:fenics-packages/fenics && \
     apt-get -qq update && \
     apt-get -qq install --no-install-recommends fenics
 USER precice
 ARG FENICS_ADAPTER_REF
 # Building fenics-adapter
-RUN python3 -m venv --system-site-packages /home/precice/venv && \
-    . /home/precice/venv/bin/activate && \
-    pip3 install git+https://github.com/precice/fenics-adapter.git@${FENICS_ADAPTER_REF}
+# Uninstall the venv's pip-installed mpi4py afterwards: FEniCS/PETSc were
+# compiled against the system mpi4py, and the venv's mpi4py (a transitive
+# dependency of fenicsprecice) causes a binary incompatibility
+# (ValueError: mpi4py.MPI.Op size changed...). Removing it lets Python fall
+# through to the system mpi4py on sys.path, while pyprecice stays in the venv.
+RUN pip3 install git+https://github.com/precice/fenics-adapter.git@${FENICS_ADAPTER_REF} && \
+    pip3 uninstall -y mpi4py
 
-FROM precice_dependecies AS fenicsx_adapter
+FROM python_bindings AS fenicsx_adapter
 COPY --from=python_bindings /home/precice/.local /home/precice/.local
 USER root
 RUN add-apt-repository -y ppa:fenics-packages/fenics && \
@@ -143,17 +148,22 @@ RUN add-apt-repository -y ppa:fenics-packages/fenics && \
 USER precice
 ARG FENICSX_ADAPTER_REF
 # Building fenicsx-adapter
-RUN python3 -m venv --system-site-packages /home/precice/venv && \
-    . /home/precice/venv/bin/activate && \
-    pip3 install git+https://github.com/precice/fenicsx-adapter.git@${FENICSX_ADAPTER_REF}
+# Uninstall the venv's pip-installed mpi4py afterwards: FEniCS/PETSc were
+# compiled against the system mpi4py, and the venv's mpi4py (a transitive
+# dependency of fenicsprecice) causes a binary incompatibility
+# (ValueError: mpi4py.MPI.Op size changed...). Removing it lets Python fall
+# through to the system mpi4py on sys.path, while pyprecice stays in the venv.
+RUN pip3 install git+https://github.com/precice/fenicsx-adapter.git@${FENICSX_ADAPTER_REF} && \
+    pip3 uninstall -y mpi4py
 
-FROM precice_dependecies AS nutils_adapter
-COPY --from=python_bindings /home/precice/.local /home/precice/.local
+FROM python_bindings AS nutils_adapter
+USER root
+# The following are dependencies of gmsh, needed by the turek-hron-fsi3 nutils participant
+RUN apt-get -qq update && \
+apt-get -qq install libglu1-mesa libxrender1 libxcursor1 libxft2 libxinerama1
 USER precice
 # Installing nutils - There is no adapter
-RUN python3 -m venv /home/precice/venv && \
-    . /home/precice/venv/bin/activate && \
-    pip3 install nutils
+RUN pip3 install nutils
 
 
 FROM precice_dependecies AS calculix_adapter
@@ -180,7 +190,6 @@ RUN git clone https://github.com/precice/calculix-adapter.git && \
     ln -s /home/precice/calculix-adapter/bin/ccx_preCICE /home/precice/.local/bin/ccx_preCICE
 
 FROM python_bindings AS su2_adapter
-COPY --from=precice /home/precice/.local /home/precice/.local
 USER root
 RUN apt-get -qq update && \
     apt-get -qq install swig python3-mpi4py python3-setuptools
@@ -193,8 +202,6 @@ WORKDIR /home/precice
 RUN wget https://github.com/su2code/SU2/archive/refs/tags/v${SU2_VERSION}.tar.gz && \
     tar xvzf v${SU2_VERSION}.tar.gz && \
     rm -fv v${SU2_VERSION}.tar.gz
-RUN python3 -m venv /home/precice/venv && \
-    . /home/precice/venv/bin/activate
 ARG SU2_ADAPTER_PR
 ARG SU2_ADAPTER_REF
 WORKDIR /home/precice
@@ -245,27 +252,22 @@ RUN git clone https://bitbucket.org/davidscn/mercurydpm.git &&\
     cmake -D MercuryDPM_PreCICE_COUPLING="ON" .. && \
     make -j $(nproc) ChannelTransport
 
-FROM precice_dependecies AS micro_manager
+FROM python_bindings AS micro_manager
 USER precice
 WORKDIR /home/precice
-COPY --from=precice /home/precice/.local/ /home/precice/.local/
 
 ARG MICRO_MANAGER_REF
-RUN python3 -m venv /home/precice/venv && \
-    . /home/precice/venv/bin/activate && \
-    git clone https://github.com/precice/micro-manager.git && \
+RUN git clone https://github.com/precice/micro-manager.git && \
     cd micro-manager && \
     if [ -n "${MICRO_MANAGER_PR}" ]; then git fetch origin pull/${MICRO_MANAGER_PR}/head; fi && \
     git checkout ${MICRO_MANAGER_REF} && \
     pip3 install .
+RUN pip3 install pybind11
+RUN pip3 install nutils
 WORKDIR /home/precice
-RUN python3 -m venv /home/precice/venv && \
-    . /home/precice/venv/bin/activate && \
-    pip3 install pybind11    
 
 FROM micro_manager AS dumux_adapter
 USER root
-COPY --from=precice /home/precice/.local/ /home/precice/.local/
 ARG DUNE_VERSION
 ARG DUMUX_VERSION
 USER precice