diff --git a/.github/workflows/test_pyodide.yml b/.github/workflows/test_pyodide.yml index 24c2adb6f..971969fcf 100644 --- a/.github/workflows/test_pyodide.yml +++ b/.github/workflows/test_pyodide.yml @@ -30,7 +30,7 @@ jobs: - name: pyodide run: - python scripts/test.py ${{matrix.args}} pyodide + python scripts/test.py ${{matrix.args}} --cibw-pyodide 1 cibw # We do not use upload-artifact@v4 because it fails due to us creating # identically-named wheels. diff --git a/docs/installation.rst b/docs/installation.rst index b4af61ce6..55c273aba 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -265,6 +265,28 @@ not generally supported: command line. +Official PyMuPDF Linux wheels may not install on older Linux systems +-------------------------------------------------------------------- + +Releases of PyMuPDF are incompatible with older Linux systems. + +For example as of 2025-09-03, `pip install pymupdf` does not work on some AWS +Lambda systems - see https://github.com/pymupdf/PyMuPDF/discussions/4631. + +This is because official PyMuPDF Linux wheels are built with a version of +glibc determined by the current Python manylinux environment. These wheels are +incompatible with Linux systems that have an older glibc. + +The official Python manylinux environment is updated periodically to use newer +glibc versions, so new releases of PyMuPDF become increasingly incompatible +with older Linux systems. + +There is nothing that can be done about this, other than updating older Linux +systems, or building PyMuPDF locally from source. + +For more details, please see: `Python Packaging Authority `_. + + Packaging --------- diff --git a/pipcl.py b/pipcl.py index 77b61797c..8589eea30 100644 --- a/pipcl.py +++ b/pipcl.py @@ -415,8 +415,11 @@ def __init__(self, added. `to_` identifies what the file should be called within a wheel - or when installing. If `to_` ends with `/`, the leaf of `from_` - is appended to it (and `from_` must not be a `bytes`). + or when installing. If `to_` is empty or `/` we set it to the + leaf of `from_` (`from_` must not be a `bytes`) - i.e. we place + the file in the root directory of the wheel; otherwise if + `to_` ends with `/` the leaf of `from_` is appended to it (and + `from_` must not be a `bytes`). Initial `$dist-info/` in `_to` is replaced by `{name}-{version}.dist-info/`; this is useful for license files @@ -1446,8 +1449,12 @@ def _fromto(self, p): `p` is a tuple `(from_, to_)` where `from_` is str/bytes and `to_` is str. If `from_` is a bytes it is contents of file to add, otherwise the path of an existing file; non-absolute paths are assumed to be relative - to `self.root`. If `to_` is empty or ends with `/`, we append the leaf - of `from_` (which must be a str). + to `self.root`. + + If `to_` is empty or `/` we set it to the leaf of `from_` (which must + be a str) - i.e. we place the file in the root directory of the wheel; + otherwise if `to_` ends with `/` we append the leaf of `from_` (which + must be a str). If `to_` starts with `$dist-info/`, we replace this with `self._dist_info_dir()`. @@ -1467,7 +1474,9 @@ def _fromto(self, p): from_, to_ = p assert isinstance(from_, (str, bytes)) assert isinstance(to_, str) - if to_.endswith('/') or to_=='': + if to_ == '/' or to_ == '': + to_ = os.path.basename(from_) + elif to_.endswith('/'): to_ += os.path.basename(from_) prefix = '$dist-info/' if to_.startswith( prefix): diff --git a/scripts/test.py b/scripts/test.py index a76bb8a3d..000f865a1 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -792,7 +792,7 @@ def build( venv, wheel, ): - print(f'{build_isolation=}') + log(f'{build_isolation=}') if build_isolation is None: # On OpenBSD libclang is not available on pypi.org, so we need to force @@ -884,7 +884,8 @@ def cibuildwheel( log(f'{CIBW_BUILD=}') if CIBW_BUILD is None: if cibw_pyodide: - CIBW_BUILD = 'cp312*' + # Using python-3.13 fixes problems with MuPDF's setjmp/longjmp. + CIBW_BUILD = 'cp313*' elif os.environ.get('GITHUB_ACTIONS') == 'true': # Build/test all supported Python versions. CIBW_BUILD = 'cp39* cp310* cp311* cp312* cp313*' @@ -939,19 +940,33 @@ def cibw_do_test_project(env_extra, CIBW_BUILD, cibw_pyodide, cibw_pyodide_args) f.write(textwrap.dedent(f''' import shutil import sys + import os import pipcl def build(): + + cc_base, _ = pipcl.base_compiler(cpp=True) + ld_base, _ = pipcl.base_linker(cpp=True) + pipcl.run(f'mkdir -p {testdir}/build') + pipcl.run(f'{{cc_base}} -DNDEBUG -fPIC -c -o {testdir}/build/qwerty.o {testdir}/qwerty.cpp') + pipcl.run(f'{{ld_base}} -o {testdir}/build/libqwerty.so {testdir}/build/qwerty.o') + so_leaf = pipcl.build_extension( name = 'foo', path_i = 'foo.i', outdir = 'build', - compiler_extra = {'-fwasm-exceptions -sSUPPORT_LONGJMP=wasm' if cibw_pyodide else ''!r}, - linker_extra = {'-fwasm-exceptions -sSUPPORT_LONGJMP=wasm' if cibw_pyodide else ''!r}, + libpaths = '{testdir}/build', + libs = ['qwerty'], ) + return [ ('build/foo.py', 'foo/__init__.py'), (f'build/{{so_leaf}}', f'foo/'), + + # 2025-09-03: formally we put libraries in foo.lib; now it seems + # they need to be at top level in wheel. + # + (f'build/libqwerty.so', f'/'), ] p = pipcl.Package( @@ -973,11 +988,17 @@ def build(): #include #include #include + + int qwerty(void); static sigjmp_buf jmpbuf; static int bar0(const char* text) { - printf("bar(): text: %s\\\\n", text); + printf("bar(): text: %s\\n", text); + + int q = qwerty(); + printf("bar(): q=%i\\n", q); + int len = (int) strlen(text); printf("bar(): len=%i\\\\n", len); //printf("calling longjmp().\\n"); @@ -1016,23 +1037,25 @@ def build(): %} int bar(const char* text); ''')) + + with open(f'{testdir}/qwerty.cpp', 'w') as f: + f.write(textwrap.dedent(''' + #include + int qwerty(void) + { + printf("qwerty()\\n"); + return 3; + } + ''')) + shutil.copy2(f'{pymupdf_dir_abs}/pipcl.py', f'{testdir}/pipcl.py') shutil.copy2(f'{pymupdf_dir_abs}/wdev.py', f'{testdir}/wdev.py') env_extra['CIBW_BUILD'] = CIBW_BUILD - env_extra['CIBW_TEST_COMMAND'] = f'python -c "import foo; foo.bar(\\"some text\\")"' + env_extra['CIBW_TEST_COMMAND'] = f'pyodide xbuildenv search --all; python -c "import foo; foo.bar(\\"some text\\")"; true' - if cibw_pyodide: - # Expect failure. - e, text = run(f'cd {testdir} && cibuildwheel{cibw_pyodide_args}', env_extra=env_extra, capture=1, check=0) - log(f'{e=}') - log(f'text is:\n{textwrap.indent(text, " ")}') - assert e - assert 'module="env" function="__c_longjmp": tag import requires a WebAssembly.Tag' in text - log(f'Have detected expected exception from Pyodide.') - else: - run(f'cd {testdir} && cibuildwheel{cibw_pyodide_args}', env_extra=env_extra) - run(f'ls -ld {testdir}/wheelhouse/*') + run(f'cd {testdir} && cibuildwheel{cibw_pyodide_args}', env_extra=env_extra) + run(f'ls -ld {testdir}/wheelhouse/*') def build_pyodide_wheel(pyodide_build_version=None): @@ -1259,14 +1282,18 @@ def getmtime(path): PYODIDE_ROOT = os.environ.get('PYODIDE_ROOT') if PYODIDE_ROOT is not None: + # We can't install packages with `pip install`; setup.py will have + # specified pytest in the wheels's , so it will be + # already installed. + # log(f'Not installing test packages because {PYODIDE_ROOT=}.') - command = f'{pytest_options} {pytest_arg} -s' + command = f'{pytest_options} {pytest_arg}' args = shlex.split(command) - print(f'{PYODIDE_ROOT=} so calling pytest.main(args).') - print(f'{command=}') - print(f'args are ({len(args)}):') + log(f'{PYODIDE_ROOT=} so calling pytest.main(args).') + log(f'{command=}') + log(f'args are ({len(args)}):') for arg in args: - print(f' {arg!r}') + log(f' {arg!r}') import pytest pytest.main(args) return @@ -1334,7 +1361,7 @@ def getmtime(path): # Always start by removing any test_*_fitz.py files. for p in glob.glob(f'{pymupdf_dir_rel}/tests/test_*_fitz.py'): - print(f'Removing {p=}') + log(f'Removing {p=}') os.remove(p) if test_fitz: # Create copies of each test file, modified to use `pymupdf` @@ -1346,7 +1373,7 @@ def getmtime(path): continue branch, leaf = os.path.split(p) p2 = f'{branch}/{leaf[:5]}fitz_{leaf[5:]}' - print(f'Converting {p=} to {p2=}.') + log(f'Converting {p=} to {p2=}.') with open(p, encoding='utf8') as f: text = f.read() text2 = re.sub("([^\'])\\bpymupdf\\b", '\\1fitz', text) diff --git a/setup.py b/setup.py index 15271aa1c..cd5299607 100755 --- a/setup.py +++ b/setup.py @@ -574,14 +574,10 @@ def get_mupdf(path=None, sha=None): windows = platform.system() == 'Windows' or platform.system().startswith('CYGWIN') msys2 = platform.system().startswith('MSYS_NT-') -pyodide_flags = '-fwasm-exceptions' - if os.environ.get('PYODIDE') == '1': if os.environ.get('OS') != 'pyodide': log('PYODIDE=1, setting OS=pyodide.') os.environ['OS'] = 'pyodide' - os.environ['XCFLAGS'] = pyodide_flags - os.environ['XCXXFLAGS'] = pyodide_flags pyodide = os.environ.get('OS') == 'pyodide' @@ -704,8 +700,8 @@ def add(flavour, from_, to_): add('d', f'{mupdf_build_dir}/libmupdf-threads.a', f'{to_dir_d}/lib/') elif pyodide: add('p', f'{mupdf_build_dir}/_mupdf.so', to_dir) - add('b', f'{mupdf_build_dir}/libmupdfcpp.so', 'PyMuPDF.libs/') - add('b', f'{mupdf_build_dir}/libmupdf.so', 'PyMuPDF.libs/') + add('b', f'{mupdf_build_dir}/libmupdfcpp.so', '/') + add('b', f'{mupdf_build_dir}/libmupdf.so', '/') else: add('p', f'{mupdf_build_dir}/_mupdf.so', to_dir) add('b', pipcl.get_soname(f'{mupdf_build_dir}/libmupdfcpp.so'), to_dir) @@ -1211,10 +1207,6 @@ def _extension_flags( mupdf_local, mupdf_build_dir, build_type): if cxxflags: compiler_extra += f' {cxxflags}' - if pyodide: - compiler_extra += f' {pyodide_flags}' - linker_extra += f' {pyodide_flags}' - return compiler_extra, linker_extra, includes, defines, optimise, debug, libpaths, libs, libraries, diff --git a/src/extra.i b/src/extra.i index fbd152052..986c88735 100644 --- a/src/extra.i +++ b/src/extra.i @@ -1,5 +1,3 @@ -%module fitz_extra - %pythoncode %{ # pylint: disable=all %} diff --git a/tests/test_codespell.py b/tests/test_codespell.py index 2c4b4d75c..5edcf61e1 100644 --- a/tests/test_codespell.py +++ b/tests/test_codespell.py @@ -12,6 +12,10 @@ def test_codespell(): ''' Check rebased Python code with codespell. ''' + if os.environ.get('PYODIDE_ROOT'): + print('test_codespell(): not running on Pyodide - cannot run child processes.') + return + if not hasattr(pymupdf, 'mupdf'): print('Not running codespell with classic implementation.') return diff --git a/tests/test_flake8.py b/tests/test_flake8.py index ddc55fece..8001cc77a 100644 --- a/tests/test_flake8.py +++ b/tests/test_flake8.py @@ -9,6 +9,10 @@ def test_flake8(): ''' Check rebased Python code with flake8. ''' + if os.environ.get('PYODIDE_ROOT'): + print('test_flake8(): not running on Pyodide - cannot run child processes.') + return + if not hasattr(pymupdf, 'mupdf'): print(f'Not running flake8 with classic implementation.') return diff --git a/tests/test_font.py b/tests/test_font.py index b7d2c223c..0aa31a8c4 100644 --- a/tests/test_font.py +++ b/tests/test_font.py @@ -83,6 +83,10 @@ def test_2608(): assert text == expected def test_fontarchive(): + if os.environ.get('PYODIDE_ROOT'): + print('test_fontarchive(): not running on Pyodide - we get ValueError: No font code \'notos\' found in pymupdf-fonts..') + return + import subprocess arch = pymupdf.Archive() css = pymupdf.css_for_pymupdf_font("notos", archive=arch, name="sans-serif") @@ -234,6 +238,10 @@ def test_3887(): def test_4457(): + if os.environ.get('PYODIDE_ROOT'): + print('test_4457(): not running on Pyodide - cannot run child processes.') + return + print() files = ( ('https://github.com/user-attachments/files/20862923/test_4457_a.pdf', 'test_4457_a.pdf', None, 4), diff --git a/tests/test_general.py b/tests/test_general.py index e12207000..8091023ef 100644 --- a/tests/test_general.py +++ b/tests/test_general.py @@ -785,6 +785,9 @@ def test_2736(): def test_subset_fonts(): + if os.environ.get('PYODIDE_ROOT'): + print('test_subset_fonts(): not running on Pyodide - ValueError: No font code \'ubuntu\' found in pymupdf-fonts.') + return """Confirm subset_fonts is working.""" if not hasattr(pymupdf, "mupdf"): print("Not testing 'test_subset_fonts' in classic.") @@ -1026,6 +1029,10 @@ def next_fd(): os.remove(oldfile) def test_cli(): + if os.environ.get('PYODIDE_ROOT'): + print('test_cli(): not running on Pyodide - cannot run child processes.') + return + if not hasattr(pymupdf, 'mupdf'): print('test_cli(): Not running on classic because of fitz_old.') return @@ -1063,6 +1070,10 @@ def test_cli_out(): Check redirection of messages and log diagnostics with environment variables PYMUPDF_LOG and PYMUPDF_MESSAGE. ''' + if os.environ.get('PYODIDE_ROOT'): + print('test_cli_out(): not running on Pyodide - cannot run child processes.') + return + if not hasattr(pymupdf, 'mupdf'): print('test_cli(): Not running on classic because of fitz_old.') return @@ -1150,6 +1161,10 @@ def test_use_python_logging(): ''' Checks pymupdf.use_python_logging(). ''' + if os.environ.get('PYODIDE_ROOT'): + print('test_cli(): not running on Pyodide - cannot run child processes.') + return + log_prefix = None if os.environ.get('PYMUPDF_USE_EXTRA') == '0': log_prefix = f'.+Using non-default setting from PYMUPDF_USE_EXTRA: \'0\'' @@ -1433,6 +1448,10 @@ def test_open2(): Checks behaviour of fz_open_document() and fz_open_document_with_stream() with different filenames/magic values. ''' + if os.environ.get('PYODIDE_ROOT'): + print('test_open2(): not running on Pyodide - cannot run child processes.') + return + if platform.system() == 'Windows': print(f'test_open2(): not running on Windows because `git ls-files` known fail on Github Windows runners.') return @@ -1789,6 +1808,10 @@ def test_4309(): document.delete_page() def test_4263(): + if os.environ.get('PYODIDE_ROOT'): + print('test_4263(): not running on Pyodide - cannot run child processes.') + return + path = os.path.normpath(f'{__file__}/../../tests/resources/test_4263.pdf') path_out = f'{path}.linerarized.pdf' command = f'pymupdf clean -linear {path} {path_out}' @@ -1915,6 +1938,10 @@ def show(items): def test_4533(): + if os.environ.get('PYODIDE_ROOT'): + print('test_4533(): not running on Pyodide - cannot run child processes.') + return + print() path = util.download( 'https://github.com/user-attachments/files/20497146/NineData_user_manual_V3.0.5.pdf', @@ -1966,6 +1993,10 @@ def test_gitinfo(): def test_4392(): + if os.environ.get('PYODIDE_ROOT'): + print('test_4392(): not running on Pyodide - cannot run child processes.') + return + print() path = os.path.normpath(f'{__file__}/../../tests/test_4392.py') with open(path, 'w') as f: diff --git a/tests/test_import.py b/tests/test_import.py index a8a8192c1..abc60865a 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -5,6 +5,10 @@ def test_import(): + if os.environ.get('PYODIDE_ROOT'): + print('test_import(): not running on Pyodide - cannot run child processes.') + return + root = os.path.abspath(f'{__file__}/../../') p = f'{root}/tests/resources_test_import.py' with open(p, 'w') as f: diff --git a/tests/test_memory.py b/tests/test_memory.py index 3985a67e3..cfaccbd92 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -17,6 +17,10 @@ def test_2791(): ''' Check for memory leaks. ''' + if os.environ.get('PYODIDE_ROOT'): + print('test_2791(): not running on Pyodide - No module named \'psutil\'.') + return + if os.environ.get('PYMUPDF_RUNNING_ON_VALGRIND') == '1': print(f'test_2791(): not running because PYMUPDF_RUNNING_ON_VALGRIND=1.') return @@ -94,6 +98,10 @@ def get_stat(): def test_4090(): + if os.environ.get('PYODIDE_ROOT'): + print('test_4090(): not running on Pyodide - No module named \'psutil\'.') + return + print(f'test_4090(): {os.environ.get("PYTHONMALLOC")=}.') import psutil process = psutil.Process() @@ -148,6 +156,10 @@ def show_tracemalloc_diff(snapshot1, snapshot2): def test_4125(): + if os.environ.get('PYODIDE_ROOT'): + print('test_4125(): not running on Pyodide - No module named \'psutil\'.') + return + if os.environ.get('PYMUPDF_RUNNING_ON_VALGRIND') == '1': print(f'test_4125(): not running because PYMUPDF_RUNNING_ON_VALGRIND=1.') return diff --git a/tests/test_pixmap.py b/tests/test_pixmap.py index 4c91bcf6c..97f0ff0ed 100644 --- a/tests/test_pixmap.py +++ b/tests/test_pixmap.py @@ -70,7 +70,7 @@ def test_pilsave(): pix2 = pymupdf.Pixmap(stream) assert repr(pix1) == repr(pix2) except ModuleNotFoundError: - assert platform.system() == 'Windows' and sys.maxsize == 2**31 - 1 + assert platform.system() in ('Windows', 'Emscripten') and sys.maxsize == 2**31 - 1 def test_save(tmpdir): @@ -556,6 +556,9 @@ def test_4423(): def test_4445(): + if os.environ.get('PYODIDE_ROOT'): + print('test_4445(): not running on Pyodide - cannot run child processes.') + return print() # Test case is large so we download it instead of having it in PyMuPDF # git. We put it in `cache/` directory do it is not removed by `git clean` diff --git a/tests/test_pylint.py b/tests/test_pylint.py index a3b48ae6f..38c6d017f 100644 --- a/tests/test_pylint.py +++ b/tests/test_pylint.py @@ -7,6 +7,10 @@ def test_pylint(): + if os.environ.get('PYODIDE_ROOT'): + print('test_pylint(): not running on Pyodide - cannot run child processes.') + return + if not hasattr(pymupdf, 'mupdf'): print(f'test_pylint(): Not running with classic implementation.') return diff --git a/tests/test_tesseract.py b/tests/test_tesseract.py index c3c9f7283..a36120cce 100644 --- a/tests/test_tesseract.py +++ b/tests/test_tesseract.py @@ -15,6 +15,10 @@ def test_tesseract(): But if TESSDATA_PREFIX is set in the environment, we assert that FzPage.get_textpage_ocr() succeeds. ''' + if os.environ.get('PYODIDE_ROOT'): + print('test_tesseract(): not running on Pyodide - cannot run child processes.') + return + path = os.path.abspath( f'{__file__}/../resources/2.pdf') doc = pymupdf.open( path) page = doc[5] @@ -71,6 +75,10 @@ def test_3842b(): # # Note that Tesseract seems to output its own diagnostics. # + if os.environ.get('PYODIDE_ROOT'): + print('test_3842b(): not running on Pyodide - cannot run child processes.') + return + path = os.path.normpath(f'{__file__}/../../tests/resources/test_3842.pdf') with pymupdf.open(path) as document: page = document[6] @@ -91,6 +99,10 @@ def test_3842b(): def test_3842(): + if os.environ.get('PYODIDE_ROOT'): + print('test_3842(): not running on Pyodide - cannot run child processes.') + return + path = os.path.normpath(f'{__file__}/../../tests/resources/test_3842.pdf') with pymupdf.open(path) as document: page = document[6] diff --git a/tests/test_textextract.py b/tests/test_textextract.py index 5883a3673..3cac90030 100644 --- a/tests/test_textextract.py +++ b/tests/test_textextract.py @@ -263,6 +263,10 @@ def test_3197(): def test_document_text(): + if os.environ.get('PYODIDE_ROOT'): + print('test_document_text(): not running on Pyodide - multiprocessing not available.') + return + import platform import time @@ -310,6 +314,9 @@ def llen(texts): def test_4524(): + if os.environ.get('PYODIDE_ROOT'): + print('test_4524(): not running on Pyodide - multiprocessing not available.') + return path = os.path.abspath(f'{__file__}/../../tests/resources/mupdf_explored.pdf') print('') document = pymupdf.Document(path)