Skip to content

Commit f31f392

Browse files
committed
Allow LD_LIBRARY_PATH override for python tests
On some HPC toolchains, a third-party Python package imported before ecflow (for example numpy) resolves and loads the platform default libstdc++ first. Since ELF soname resolution is process-wide and first-one-wins, this can leave ecflow.so unable to satisfy the GLIBCXX symbol version it was actually built against, when the build toolchain differs from the platform default. Add ECFLOW_PYEXT_TEST_LD_LIBRARY_PATH, a configure-time cache variable that prepends a given runtime library directory to LD_LIBRARY_PATH for the py3_u_* and py3_s_* tests, so the correct libstdc++.so.6 is resolved regardless of which component asks for it first. Left empty by default, leaving unaffected platforms untouched. Wire the new flag into the gnu-15.2.0 and gnu-14.2.0 HPC CI jobs in .github/ci-hpc-config.yml, pointing at each toolchain's libstdc++ directory, so the affected python tests actually pass in CI.
1 parent 0ae7393 commit f31f392

2 files changed

Lines changed: 37 additions & 4 deletions

File tree

.github/ci-hpc-config.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ gnu-15.2.0:
3737
-DPython3_LIBRARIES=/usr/local/apps/python3/3.13.13-01/lib64/libpython3.13.so
3838
-DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.13.13-01/lib64
3939
-DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.13.13-01/lib/python3.13/site-packages/pybind11/share/cmake/pybind11
40+
-DECFLOW_PYEXT_TEST_LD_LIBRARY_PATH=/usr/local/apps/gcc/15.2.0/lib64
4041
# Note: Python 3.13.13-01, provides Pybind11 3.0.4.
4142
ctest_options:
4243
# disable s_http* tests
@@ -65,6 +66,7 @@ gnu-14.2.0:
6566
-DPython3_LIBRARIES=/usr/local/apps/python3/3.12.9-01/lib64/libpython3.12.so
6667
-DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.12.9-01/lib64
6768
-DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.13.13-01/lib/python3.13/site-packages/pybind11/share/cmake/pybind11
69+
-DECFLOW_PYEXT_TEST_LD_LIBRARY_PATH=/usr/local/apps/gcc/14.2.0/lib64
6870
# Note: Python 3.13.13-01, provides Pybind11 3.0.4.
6971
#
7072
# Important:

libs/pyext/python3/CMakeLists.txt

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,16 +68,42 @@ set_target_properties(ecflow3
6868
# with 'No module named pytest'.
6969
execute_process(
7070
COMMAND ${Python3_EXECUTABLE} -c "import pytest"
71-
RESULT_VARIABLE _ecflow_have_pytest
71+
RESULT_VARIABLE _ECFLOW_HAVE_PYTEST
7272
OUTPUT_QUIET ERROR_QUIET)
73-
if( NOT _ecflow_have_pytest EQUAL 0 )
73+
if( NOT _ECFLOW_HAVE_PYTEST EQUAL 0 )
7474
message( WARNING
7575
"pytest was not found for ${Python3_EXECUTABLE}; the python tests "
7676
"(py3_u_* and py3_s_*) will fail to run. Install it with:\n"
7777
" ${Python3_EXECUTABLE} -m pip install -r "
7878
"${CMAKE_CURRENT_SOURCE_DIR}/../test/requirements-test.txt" )
7979
endif()
8080

81+
#
82+
# When using some HPC toolchains, the dynamic linker for ecflow resolves and loads the platform default libstdc++ first,
83+
# instead of the one that ecflow.so was built against. This leads to symbol version mismatches, and the python tests
84+
# fail with errors like:
85+
#
86+
# /lib64/libstdc++.so.6: error: version lookup error: version `GLIBCXX_3.4.32' not found
87+
# (required by /path/to/ecflow/build/libs/pyext/python3/ecflow.so) (fatal)
88+
#
89+
# ECFLOW_PYEXT_TEST_LD_LIBRARY_PATH allows a runtime library directory to be prepended to LD_LIBRARY_PATH, so that
90+
# the correct libstdc++.so.6 is used by the python tests.
91+
#
92+
set(ECFLOW_PYEXT_TEST_LD_LIBRARY_PATH "" CACHE STRING
93+
"Colon-separated directories prepended to LD_LIBRARY_PATH when running the python tests")
94+
95+
set( _ECFLOW_PYEXT_ENVIRONMENT
96+
"PATH=${Python3_EXECUTABLE_DIR}:$ENV{PATH}"
97+
"PYTHONPATH=${ECFLOW_PYTHONPATH}"
98+
)
99+
if( ECFLOW_PYEXT_TEST_LD_LIBRARY_PATH )
100+
if( DEFINED ENV{LD_LIBRARY_PATH} AND NOT "$ENV{LD_LIBRARY_PATH}" STREQUAL "" )
101+
list( APPEND _ECFLOW_PYEXT_ENVIRONMENT "LD_LIBRARY_PATH=${ECFLOW_PYEXT_TEST_LD_LIBRARY_PATH}:$ENV{LD_LIBRARY_PATH}" )
102+
else()
103+
list( APPEND _ECFLOW_PYEXT_ENVIRONMENT "LD_LIBRARY_PATH=${ECFLOW_PYEXT_TEST_LD_LIBRARY_PATH}" )
104+
endif()
105+
endif()
106+
81107
foreach( test ${u_tests} )
82108
ecbuild_add_test(
83109
TARGET
@@ -88,7 +114,7 @@ foreach( test ${u_tests} )
88114
ARGS
89115
-m pytest ${CMAKE_CURRENT_SOURCE_DIR}/../test/py_${test}.py -v
90116
ENVIRONMENT
91-
"PATH=${Python3_EXECUTABLE_DIR}:$ENV{PATH};PYTHONPATH=${ECFLOW_PYTHONPATH}"
117+
${_ECFLOW_PYEXT_ENVIRONMENT}
92118
TEST_DEPENDS
93119
u_base
94120
)
@@ -97,6 +123,11 @@ endforeach()
97123

98124
if ( ENABLE_ALL_TESTS AND ENABLE_SERVER)
99125

126+
#
127+
# Python integration tests run the ecflow server, and thus use ECF_SSL_DIR to find the generated SSL certificates.
128+
#
129+
list( APPEND _ECFLOW_PYEXT_ENVIRONMENT "ECF_SSL_DIR=${CMAKE_CURRENT_BINARY_DIR}/" )
130+
100131
foreach( test ${s_tests} )
101132
ecbuild_add_test(
102133
TARGET
@@ -107,7 +138,7 @@ if ( ENABLE_ALL_TESTS AND ENABLE_SERVER)
107138
ARGS
108139
-m pytest ${CMAKE_CURRENT_SOURCE_DIR}/../test/py_${test}.py -v
109140
ENVIRONMENT
110-
"PATH=${Python3_EXECUTABLE_DIR}:$ENV{PATH};PYTHONPATH=${ECFLOW_PYTHONPATH};ECF_SSL_DIR=${CMAKE_CURRENT_BINARY_DIR}/"
141+
${_ECFLOW_PYEXT_ENVIRONMENT}
111142
TEST_DEPENDS
112143
u_base
113144
)

0 commit comments

Comments
 (0)