From 9ad7cb0f5076b9cfd33db605ef880b5226429c61 Mon Sep 17 00:00:00 2001 From: Julian Smith Date: Wed, 16 Jul 2025 15:53:43 +0100 Subject: [PATCH 01/10] scripts/test.py: Allow cibuildwheel to build and test pyodide. We can't run child processes on Pyodide, so for testing we import pytest and call pytest.main(). --- scripts/test.py | 60 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/scripts/test.py b/scripts/test.py index 340d10829..2627073a9 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -78,6 +78,9 @@ 2025-05-27: this fails when building mupdf C API - `ld -r -b binary ...` fails with: emcc: error: binary: No such file or directory ("binary" was expected to be an input file, based on the commandline arguments provided) + + --cibw-pyodide-version + Override default Pyodide version to use with `cibuildwheel` command. --cibw-release-1 Set up so that `cibw` builds all wheels except linux-aarch64, and sdist @@ -167,8 +170,9 @@ inside C++ pybind. Requires `sudo apt install pybind11-dev` or similar. --pyodide-build-version - Version of Python package pyodide-build; if None (the default) we use - latest available version. + Version of Python package pyodide-build to use with `pyodide` command. + + If None (the default) `pyodide` uses the latest available version. 2025-02-13: pyodide_build_version='0.29.3' works. -s 0 | 1 @@ -296,8 +300,9 @@ def main(argv): return build_isolation = None - cibw_name = 'cibuildwheel' + cibw_name = None cibw_pyodide = None + cibw_pyodide_version = None commands = list() env_extra = dict() implementations = 'r' @@ -354,6 +359,9 @@ def main(argv): elif arg == '--build-isolation': build_isolation = int(next(args)) + elif arg == '--cibw-pyodide-version': + cibw_pyodide_version = next(args) + elif arg == '--cibw-release-1': cibw_sdist = True env_extra['CIBW_ARCHS_LINUX'] = 'auto64' @@ -531,7 +539,17 @@ def main(argv): elif command == 'cibw': # Build wheel(s) with cibuildwheel. - cibuildwheel(env_extra, cibw_name, cibw_pyodide, cibw_sdist) + if cibw_pyodide and env_extra.get('CIBW_BUILD') is None: + CIBW_BUILD = 'cp313*' + env_extra['CIBW_BUILD'] = CIBW_BUILD + log(f'Defaulting to {CIBW_BUILD=} for Pyodide.') + cibuildwheel( + env_extra, + cibw_name or 'cibuildwheel', + cibw_pyodide, + cibw_pyodide_version or '0.28.0', + cibw_sdist, + ) elif command == 'install': p = 'pymupdf' @@ -671,7 +689,7 @@ def build( run(f'pip install{build_isolation_text} -v --force-reinstall {pymupdf_dir_abs}', env_extra=env_extra) -def cibuildwheel(env_extra, cibw_name, cibw_pyodide, cibw_sdist): +def cibuildwheel(env_extra, cibw_name, cibw_pyodide, cibw_pyodide_version, cibw_sdist): if cibw_sdist and platform.system() == 'Linux': log(f'Building sdist.') @@ -727,6 +745,17 @@ def cibuildwheel(env_extra, cibw_name, cibw_pyodide, cibw_sdist): v = platform.python_version_tuple()[:2] log(f'{v=}') CIBW_BUILD = f'cp{"".join(v)}*' + + cibw_pyodide_args = '' + if cibw_pyodide: + cibw_pyodide_args = ' --platform pyodide' + env_extra['HAVE_LIBCRYPTO'] = 'no' + env_extra['PYMUPDF_SETUP_MUPDF_TESSERACT'] = '0' + if cibw_pyodide_version: + # 2025-07-21: there is no --pyodide-version option so we set + # CIBW_PYODIDE_VERSION. + env_extra['CIBW_PYODIDE_VERSION'] = cibw_pyodide_version + env_extra['CIBW_ENABLE'] = 'pyodide-prerelease' # Pass all the environment variables we have set, to Linux # docker. Note that this will miss any settings in the original @@ -735,18 +764,17 @@ def cibuildwheel(env_extra, cibw_name, cibw_pyodide, cibw_sdist): # Build for lowest (assumed first) Python version. # - cibw_pyodide_arg = ' --platform pyodide' if cibw_pyodide else '' CIBW_BUILD_0 = CIBW_BUILD.split()[0] log(f'Building for first Python version {CIBW_BUILD_0}.') env_extra['CIBW_BUILD'] = CIBW_BUILD_0 - run(f'cd {pymupdf_dir} && cibuildwheel{cibw_pyodide_arg}', env_extra=env_extra) + run(f'cd {pymupdf_dir} && cibuildwheel{cibw_pyodide_args}', env_extra=env_extra) # Tell cibuildwheel to build and test all specified Python versions; it # will notice that the wheel we built above supports all versions of # Python, so will not actually do any builds here. # env_extra['CIBW_BUILD'] = CIBW_BUILD - run(f'cd {pymupdf_dir} && cibuildwheel{cibw_pyodide_arg}', env_extra=env_extra) + run(f'cd {pymupdf_dir} && cibuildwheel{cibw_pyodide_args}', env_extra=env_extra) run(f'ls -ld {pymupdf_dir}/wheelhouse/*') @@ -972,6 +1000,20 @@ def getmtime(path): python = gh_release.relpath(sys.executable) log('Running tests with tests/run_compound.py and pytest.') + PYODIDE_ROOT = os.environ.get('PYODIDE_ROOT') + if PYODIDE_ROOT is not None: + log(f'Not installing test packages because {PYODIDE_ROOT=}.') + command = f'{pytest_options} {pytest_arg} -s' + args = shlex.split(command) + print(f'{PYODIDE_ROOT=} so calling pytest.main(args).') + print(f'{command=}') + print(f'args are ({len(args)}):') + for arg in args: + print(f' {arg!r}') + import pytest + pytest.main(args) + return + if venv == 2: run(f'pip install --upgrade {gh_release.test_packages}') else: @@ -1060,7 +1102,7 @@ def getmtime(path): try: log(f'Running tests with tests/run_compound.py and pytest.') run(command, env_extra=env_extra, timeout=test_timeout) - + except subprocess.TimeoutExpired as e: log(f'Timeout when running tests.') raise From 74c0e344d348f1ef92793935eec1dd5329f12f2b Mon Sep 17 00:00:00 2001 From: Julian Smith Date: Wed, 16 Jul 2025 17:22:59 +0100 Subject: [PATCH 02/10] setup.py: Support cibuildwheel with Pyodide. We need to specify pytest in requires_dist if on pyodide, because we cannot `pip install` it ourselves because child processes are not supported. Set extra compile/link flags for pyodide, including in XCFLAGS and XCXXFLAGS, for building MuPDF. --- setup.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e8c5b2248..f8356281a 100755 --- a/setup.py +++ b/setup.py @@ -583,14 +583,17 @@ def get_mupdf(path=None, sha=None): windows = platform.system() == 'Windows' or platform.system().startswith('CYGWIN') msys2 = platform.system().startswith('MSYS_NT-') +pyodide_flags = '-fwasm-exceptions' + if os.environ.get('PYODIDE') == '1': if os.environ.get('OS') != 'pyodide': log('PYODIDE=1, setting OS=pyodide.') os.environ['OS'] = 'pyodide' + os.environ['XCFLAGS'] = pyodide_flags + os.environ['XCXXFLAGS'] = pyodide_flags pyodide = os.environ.get('OS') == 'pyodide' - def build(): ''' pipcl.py `build_fn()` callback. @@ -1180,6 +1183,10 @@ def _extension_flags( mupdf_local, mupdf_build_dir, build_type): if cxxflags: compiler_extra += f' {cxxflags}' + if pyodide: + compiler_extra += f' {pyodide_flags}' + linker_extra += f' {pyodide_flags}' + return compiler_extra, linker_extra, includes, defines, optimise, debug, libpaths, libs, libraries, @@ -1291,7 +1298,7 @@ def get_requires_for_build_wheel(config_settings=None): readme_d = f.read() tag_python = None - requires_dist = None, + requires_dist = list() entry_points = None if 'p' in PYMUPDF_SETUP_FLAVOUR: @@ -1300,7 +1307,7 @@ def get_requires_for_build_wheel(config_settings=None): readme = readme_p summary = 'A high performance Python library for data extraction, analysis, conversion & manipulation of PDF (and other) documents.' if 'b' not in PYMUPDF_SETUP_FLAVOUR: - requires_dist = f'PyMuPDFb =={version_b}' + requires_dist.append(f'PyMuPDFb =={version_b}') # Create a `pymupdf` command. entry_points = textwrap.dedent(''' [console_scripts] @@ -1320,6 +1327,10 @@ def get_requires_for_build_wheel(config_settings=None): tag_python = 'py3' else: assert 0, f'Unrecognised {PYMUPDF_SETUP_FLAVOUR=}.' + + if os.environ.get('PYODIDE_ROOT'): + # We can't pip install pytest on pyodide, so specify it here. + requires_dist.append('pytest') p = pipcl.Package( name, From 3ec8db5ced0c87c526bbfcb28d0183046a420699 Mon Sep 17 00:00:00 2001 From: Julian Smith Date: Wed, 23 Jul 2025 10:58:52 +0100 Subject: [PATCH 03/10] pipcl.py: add support for building on graal. Graal builds work by deferring to a non-graal build with graal-python's include and library paths. Other: run(): Added arg, output with prefix for each line. Added arg so caller can use a subset of os.environ. Removed arg - we always use text command, not list of args. Use run()'s when running some commands. build_extension(): allow swig=None to default to 'swig'. git_get(): allow 'git:' string to override/specify remote/branch/tag. PythonFlags: Added experimental code using sysconfig.*() instead of python-config command. swig_get(): new, useful when wanting to use latest/specific swig. --- pipcl.py | 324 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 302 insertions(+), 22 deletions(-) diff --git a/pipcl.py b/pipcl.py index 4a3edd0e1..566ac1798 100644 --- a/pipcl.py +++ b/pipcl.py @@ -11,9 +11,14 @@ that a `setup.py` script can take over the details itself. Run doctests with: `python -m doctest pipcl.py` + +For Graal we require that PIPCL_GRAAL_PYTHON is set to non-graal Python (we +build for non-graal except with Graal Python's include paths and library +directory). ''' import base64 +import codecs import glob import hashlib import inspect @@ -572,6 +577,11 @@ def assert_str_or_multi( v): self.wheel_compression = wheel_compression self.wheel_compresslevel = wheel_compresslevel + + # If true and we are building for graal, we set PIPCL_PYTHON_CONFIG to + # a command that will print includes/libs from graal_py's sysconfig. + # + self.graal_legacy_python_config = True def build_wheel(self, @@ -592,6 +602,59 @@ def build_wheel(self, f' metadata_directory={metadata_directory!r}' ) + if sys.implementation.name == 'graalpy': + # We build for Graal by building a native Python wheel with Graal + # Python's include paths and library directory. We then rename the + # wheel to contain graal's tag etc. + # + log0(f'### Graal build: deferring to cpython.') + python_native = os.environ.get('PIPCL_GRAAL_PYTHON') + assert python_native, f'Graal build requires that PIPCL_GRAAL_PYTHON is set.' + env_extra = dict( + PIPCL_SYSCONFIG_PATH_include = sysconfig.get_path('include'), + PIPCL_SYSCONFIG_PATH_platinclude = sysconfig.get_path('platinclude'), + PIPCL_SYSCONFIG_CONFIG_VAR_LIBDIR = sysconfig.get_config_var('LIBDIR'), + ) + # Tell native build to run pipcl.py itself to get python-config + # information about include paths etc. + if self.graal_legacy_python_config: + env_extra['PIPCL_PYTHON_CONFIG'] = f'{python_native} {os.path.abspath(__file__)} --graal-legacy-python-config' + + # Create venv. + venv_name = os.environ.get('PIPCL_GRAAL_NATIVE_VENV') + if venv_name: + log1(f'Graal using pre-existing {venv_name=}') + else: + venv_name = 'venv-pipcl-graal-native' + run(f'{shlex.quote(python_native)} -m venv {venv_name}') + log1(f'Graal using {venv_name=}') + + newfiles = NewFiles(f'{wheel_directory}/*.whl') + run( + f'. {venv_name}/bin/activate && python setup.py --dist-dir {shlex.quote(wheel_directory)} bdist_wheel', + env_extra = env_extra, + prefix = f'pipcl.py graal {python_native}: ', + ) + wheel = newfiles.get_one() + wheel_leaf = os.path.basename(wheel) + python_major_minor = run(f'{shlex.quote(python_native)} -c "import platform; import sys; sys.stdout.write(str().join(platform.python_version_tuple()[:2]))"', capture=1) + cpabi = f'cp{python_major_minor}-abi3' + assert cpabi in wheel_leaf, f'Expected wheel to be for {cpabi=}, but {wheel=}.' + graalpy_ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') + log1(f'{graalpy_ext_suffix=}') + m = re.match(r'\.graalpy(\d+[^\-]*)-(\d+)', graalpy_ext_suffix) + gpver = m[1] + cpver = m[2] + graalpy_wheel_tag = f'graalpy{cpver}-graalpy{gpver}_{cpver}_native' + name = wheel_leaf.replace(cpabi, graalpy_wheel_tag) + destination = f'{wheel_directory}/{name}' + log0(f'### Graal build: copying {wheel=} to {destination=}') + # Copying results in two wheels which appears to confuse pip, showing: + # Found multiple .whl files; unspecified behaviour. Will call build_wheel. + os.rename(wheel, destination) + log1(f'Returning {name=}.') + return name + wheel_name = self.wheel_name() path = f'{wheel_directory}/{wheel_name}' @@ -1402,7 +1465,7 @@ def build_extension( debug=False, compiler_extra='', linker_extra='', - swig='swig', + swig=None, cpp=True, prerequisites_swig=None, prerequisites_compile=None, @@ -1453,7 +1516,7 @@ def build_extension( linker_extra: Extra linker flags. Can be None. swig: - Base swig command. + Swig command; if false we use 'swig'. cpp: If true we tell SWIG to generate C++ code instead of C. prerequisites_swig: @@ -1503,6 +1566,8 @@ def build_extension( linker_extra = '' if builddir is None: builddir = outdir + if not swig: + swig = 'swig' includes_text = _flags( includes, '-I') defines_text = _flags( defines, '-D') libpaths_text = _flags( libpaths, '/LIBPATH:', '"') if windows() else _flags( libpaths, '-L') @@ -1912,37 +1977,78 @@ def git_get( tag=None, update=True, submodules=True, + default_remote=None, ): ''' Ensures that is a git checkout (at either , or HEAD) of a remote repository. - Exactly one of and must be specified. + Exactly one of and must be specified, or must start + with 'git:' and match the syntax described below. Args: remote: Remote git repostitory, for example 'https://github.com/ArtifexSoftware/mupdf.git'. + + If starts with 'git:', the remaining text should be a command-line + style string containing some or all of these args: + --branch + --tag + + These overrides , and . + + For example these all clone/update/branch master of https://foo.bar/qwerty.git to local + checkout 'foo-local': + + git_get('https://foo.bar/qwerty.git', 'foo-local', branch='master') + git_get('git:--branch master https://foo.bar/qwerty.git', 'foo-local') + git_get('git:--branch master', 'foo-local', default_remote='https://foo.bar/qwerty.git') + git_get('git:', 'foo-local', branch='master', default_remote='https://foo.bar/qwerty.git') + local: Local directory. If /.git exists, we attempt to run `git update` in it. branch: - Branch to use. + Branch to use. Is used as default if remote starts with 'git:'. depth: Depth of local checkout when cloning and fetching, or None. env_extra: Dict of extra name=value environment variables to use whenever we run git. tag: - Tag to use. + Tag to use. Is used as default if remote starts with 'git:'. update: If false we do not update existing repository. Might be useful if testing without network access. submodules: If true, we clone with `--recursive --shallow-submodules` and run `git submodule update --init --recursive` before returning. + default_remote: + The remote URL if starts with 'git:' but does not specify + the remote URL. ''' log0(f'{remote=} {local=} {branch=} {tag=}') + if remote.startswith('git:'): + remote0 = remote + args = iter(shlex.split(remote0[len('git:'):])) + remote = default_remote + while 1: + try: + arg = next(args) + except StopIteration: + break + if arg == '--branch': + branch = next(args) + tag = None + elif arg == '--tag': + tag == next(args) + branch = None + else: + remote = next(args) + assert remote, f'{default_remote=} and no remote specified in remote={remote0!r}.' + assert branch or tag, f'{branch=} {tag=} and no branch/tag specified in remote={remote0!r}.' + assert (branch and not tag) or (not branch and tag), f'Must specify exactly one of and .' depth_arg = f' --depth {depth}' if depth else '' @@ -2007,9 +2113,11 @@ def run( capture=False, check=1, verbose=1, + env=None, env_extra=None, timeout=None, caller=1, + prefix=None, ): ''' Runs a command using `subprocess.run()`. @@ -2032,12 +2140,22 @@ def run( command's returncode in our return value. verbose: If true we show the command. + env: + None or dict to use instead of . env_extra: - None or dict to add to environ. + None or dict to add to or . timeout: If not None, timeout in seconds; passed directly to subprocess.run(). Note that on MacOS subprocess.run() seems to leave processes running if timeout expires. + prefix: + String prefix for each line of output. + + If true: + * We run command with stdout=subprocess.PIPE and + stderr=subprocess.STDOUT, repetaedly reading the command's output + and writing it to stdout with . + * We do not support , which must be None. Returns: check capture Return -------------------------- @@ -2046,9 +2164,11 @@ def run( true false None or raise exception true true output or raise exception ''' - env = None + if env is None: + env = os.environ + if env_extra: + env = env.copy() if env_extra: - env = os.environ.copy() env.update(env_extra) lines = _command_lines( command) if verbose: @@ -2061,16 +2181,59 @@ def run( log1(text, caller=caller+1) sep = ' ' if windows() else ' \\\n' command2 = sep.join( lines) - cp = subprocess.run( - command2, - shell=True, - stdout=subprocess.PIPE if capture else None, - stderr=subprocess.STDOUT if capture else None, - check=check, - encoding='utf8', - env=env, - timeout=timeout, - ) + + if prefix: + assert not timeout, f'Timeout not supported with prefix.' + child = subprocess.Popen( + command2, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding='utf8', + env=env, + ) + if capture: + capture_text = '' + decoder = codecs.getincrementaldecoder('utf8')('replace') + line_start = True + while 1: + raw = os.read( child.stdout.fileno(), 10000) + text = decoder.decode(raw, final=not raw) + if text: + if capture: + capture_text += text + lines = text.split('\n') + for i, line in enumerate(lines): + if line_start: + sys.stdout.write(prefix) + line_start = False + sys.stdout.write(line) + if i < len(lines) - 1: + sys.stdout.write('\n') + line_start = True + sys.stdout.flush() + if not raw: + break + if not line_start: + sys.stdout.write('\n') + e = child.wait() + if check and e: + raise subprocess.CalledProcessError(e, command2, capture_text if capture else None) + if check: + return capture_text if capture else None + else: + return (e, capture_text) if capture else e + else: + cp = subprocess.run( + command2, + shell=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.STDOUT if capture else None, + check=check, + encoding='utf8', + env=env, + timeout=timeout, + ) if check: return cp.stdout if capture else None else: @@ -2108,9 +2271,11 @@ def log(text): log(f'{os.getcwd()=}') log(f'{platform.machine()=}') log(f'{platform.platform()=}') + log(f'{platform.python_implementation()=}') log(f'{platform.python_version()=}') log(f'{platform.system()=}') - log(f'{platform.uname()=}') + if sys.implementation.name != 'graalpy': + log(f'{platform.uname()=}') log(f'{sys.executable=}') log(f'{sys.version=}') log(f'{sys.version_info=}') @@ -2143,8 +2308,23 @@ class PythonFlags: String containing linker flags for library paths. ''' def __init__(self): + + # Experimental detection of python flags from sysconfig.*() instead of + # python-config command. + includes_, ldflags_ = sysconfig_python_flags() + + if pyodide(): + _include_dir = os.environ[ 'PYO3_CROSS_INCLUDE_DIR'] + _lib_dir = os.environ[ 'PYO3_CROSS_LIB_DIR'] + self.includes = f'-I {_include_dir}' + self.ldflags = f'-L {_lib_dir}' - if windows(): + elif 0: + + self.includes = includes_ + self.ldflags = ldflags_ + + elif windows(): wp = wdev.WindowsPython() self.includes = f'/I"{wp.include}"' self.ldflags = f'/LIBPATH:"{wp.libs}"' @@ -2213,8 +2393,10 @@ def __init__(self): log2(f'### Have removed `-lcrypt` from ldflags: {self.ldflags!r} -> {ldflags2!r}') self.ldflags = ldflags2 - log2(f'{self.includes=}') - log2(f'{self.ldflags=}') + log1(f'{self.includes=}') + log1(f' {includes_=}') + log1(f'{self.ldflags=}') + log1(f' {ldflags_=}') def macos_add_cross_flags(command): @@ -2766,3 +2948,101 @@ def _items(self): if os.path.isfile(path): ret[path] = self._file_id(path) return ret + + +def swig_get(swig, quick, swig_local='pipcl-swig-git'): + ''' + Returns or a new swig binary. + + If is true and starts with 'git:' (not Windows), the remaining text + is passed to git_get() and we clone/update/build swig, and return the built + binary. We default to the main swig repository, branch master, so for + example 'git:' will return the latest swig from branch master. + + Otherwise we simply return . + + Args: + swig: + If starts with 'git:', passed as arg to git_remote(). + quick: + If true, we do not update/build local checkout if the binary is + already present. + swig_local: + path to use for checkout. + ''' + if swig and swig.startswith('git:'): + assert platform.system() != 'Windows' + swig_local = os.path.abspath(swig_local) + swig_binary = f'{swig_local}/install/bin/swig' + if quick and os.path.isfile(swig_binary): + log1(f'{quick=} and {swig_binary=} already exists, so not downloading/building.') + else: + # Clone swig. + git_get( + swig, + swig_local, + default_remote='https://github.com/swig/swig.git', + branch='master', + ) + # Build swig. + run(f'cd {swig_local} && ./autogen.sh && ./configure --prefix={swig_local}/install && make && make install') + assert os.path.isfile(swig_binary) + return swig_binary + else: + return swig + + +def _show_dict(d): + ret = '' + for n in sorted(d.keys()): + v = d[n] + ret += f' {n}: {v!r}\n' + return ret + +def show_sysconfig(): + ''' + Shows contents of sysconfig.get_paths() and sysconfig.get_config_vars() dicts. + ''' + import sysconfig + paths = sysconfig.get_paths() + log0(f'show_sysconfig().') + log0(f'sysconfig.get_paths():\n{_show_dict(sysconfig.get_paths())}') + log0(f'sysconfig.get_config_vars():\n{_show_dict(sysconfig.get_config_vars())}') + + +def sysconfig_python_flags(): + ''' + Returns include paths and library directory for Python. + + Uses sysconfig.*(), overridden by environment variables + PIPCL_SYSCONFIG_PATH_include, PIPCL_SYSCONFIG_PATH_platinclude and + PIPCL_SYSCONFIG_CONFIG_VAR_LIBDIR if set. + ''' + include1_ = os.environ.get('PIPCL_SYSCONFIG_PATH_include') or sysconfig.get_path('include') + include2_ = os.environ.get('PIPCL_SYSCONFIG_PATH_platinclude') or sysconfig.get_path('platinclude') + ldflags_ = os.environ.get('PIPCL_SYSCONFIG_CONFIG_VAR_LIBDIR') or sysconfig.get_config_var('LIBDIR') + + includes_ = [include1_] + if include2_ != include1_: + includes_.append(include2) + if windows(): + includes_ = [f'/I"{i}"' for i in includes_] + ldflags_ = f'/LIBPATH:"{ldflags_}"' + else: + includes_ = [f'-I {i}' for i in includes_] + ldflags_ = f'-L {ldflags_}' + includes_ = ' '.join(includes_) + return includes_, ldflags_ + + +if __name__ == '__main__': + # Internal-only limited command line support, used if + # graal_legacy_python_config is true. + # + includes, ldflags = sysconfig_python_flags() + if sys.argv[1:] == ['--graal-legacy-python-config', '--includes']: + print(includes) + elif sys.argv[1:] == ['--graal-legacy-python-config', '--ldflags']: + print(ldflags) + else: + assert 0, f'Expected `--graal-legacy-python-config --includes|--ldflags` but {sys.argv=}' From a7fa731e3ceea9b84f69de73ea878b515ead41d0 Mon Sep 17 00:00:00 2001 From: Julian Smith Date: Wed, 23 Jul 2025 10:59:45 +0100 Subject: [PATCH 04/10] scripts/test.py: add support for graal. See new arg `--graal`. Also: * support venv=3 - delete any existing venv. * support clean mupdf builds. * support custom swig command. --- scripts/test.py | 120 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 105 insertions(+), 15 deletions(-) diff --git a/scripts/test.py b/scripts/test.py index 2627073a9..7ebc06e33 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -80,7 +80,8 @@ emcc: error: binary: No such file or directory ("binary" was expected to be an input file, based on the commandline arguments provided) --cibw-pyodide-version - Override default Pyodide version to use with `cibuildwheel` command. + Override default Pyodide version to use with `cibuildwheel` command. If + empty string with use cibuildwheel's default. --cibw-release-1 Set up so that `cibw` builds all wheels except linux-aarch64, and sdist @@ -106,6 +107,17 @@ --gdb 0|1 Run tests under gdb. Requires user interaction. + --graal + Use graal - run inside a Graal VM instead of a Python venv. + + As of 2025-08-04 we: + * Clone the latest pyenv and build it. + * Use pyenv to install graalpy. + * Use graalpy to create venv. + + [After the first time, suggest `-v 1` to avoid delay from + updating/building pyenv and recreating the graal venv.] + --help -h Show help. @@ -143,6 +155,8 @@ -m "git:--branch 1.26.x https://github.com/ArtifexSoftware/mupdf.git" -m :1.26.x + --mupdf-clean 0|1 + If 1 we do a clean MuPDF build. -M 0|1 --build-mupdf 0|1 @@ -191,6 +205,25 @@ --system-site-packages 0|1 If 1, use `--system-site-packages` when creating venv. Defaults is 0. + --swig + Use instead of the `swig` command. + + Unix only: + Clone/update/build swig from a git repository using 'git:' prefix. + + We default to https://github.com/swig/swig.git branch master, so these + are all equivalent: + + --swig 'git:--branch master https://github.com/swig/swig.git' + --swig 'git:--branch master' + --swig git: + + --swig-quick 0|1 + If 1 and `--swig` starts with 'git:', we do not update/build swig if + already present. + + See description of PYMUPDF_SETUP_SWIG_QUICK in setup.py. + -t Pytest test names, comma-separated. Should be relative to PyMuPDF directory. For example: @@ -208,12 +241,14 @@ helgrind vagrind - -v 0|1|2 + -v + venv is: 0 - do not use a venv. 1 - Use venv. If it already exists, we assume the existing directory was created by us earlier and is a valid venv containing all necessary packages; this saves a little time. - 2 - Use venv + 2 - Use venv. + 3 - Use venv but delete it first if it already exists. The default is 2. Commands: @@ -268,6 +303,7 @@ import platform import re import shlex +import shutil import subprocess import sys import textwrap @@ -305,6 +341,7 @@ def main(argv): cibw_pyodide_version = None commands = list() env_extra = dict() + graal = False implementations = 'r' install_version = None mupdf_sync = None @@ -319,6 +356,8 @@ def main(argv): show_help = False sync_paths = False system_site_packages = False + swig = None + swig_quick = None test_fitz = False test_names = list() test_timeout = None @@ -401,6 +440,9 @@ def main(argv): elif arg == '-f': test_fitz = int(next(args)) + elif arg == '--graal': + graal = True + elif arg in ('-h', '--help'): show_help = True @@ -428,6 +470,9 @@ def main(argv): os.environ['PYMUPDF_SETUP_MUPDF_BUILD'] = os.path.abspath(_mupdf) mupdf_sync = _mupdf + elif arg == '--mupdf-clean': + env_extra['PYMUPDF_SETUP_MUPDF_CLEAN']=next(args) + elif arg in ('-M', '--build-mupdf'): env_extra['PYMUPDF_SETUP_MUPDF_REBUILD'] = next(args) @@ -459,6 +504,12 @@ def main(argv): elif arg == '--system-site-packages': system_site_packages = int(next(args)) + elif arg == '--swig': + swig = next(args) + + elif arg == '--swig-quick': + swig_quick = int(next(args)) + elif arg == '-t': test_names += next(args).split(',') @@ -472,7 +523,7 @@ def main(argv): elif arg == '-v': venv = int(next(args)) - assert venv in (0, 1, 2), f'Invalid {venv=} should be 0, 1 or 2.' + assert venv in (0, 1, 2, 3), f'Invalid {venv=} should be 0, 1, 2 or 3.' elif arg in ('build', 'cibw', 'install', 'pyodide', 'test', 'wheel'): commands.append(arg) @@ -511,20 +562,50 @@ def main(argv): if venv: # Rerun ourselves inside a venv if not already in a venv. if not venv_in(): - e = venv_run( - sys.argv, - f'venv-pymupdf-{platform.python_version()}-{int.bit_length(sys.maxsize+1)}', - recreate=(venv==2), - ) + if graal: + # 2025-07-24: We need the latest pyenv. + graalpy = 'graalpy-24.2.1' + venv_name = f'venv-pymupdf-{graalpy}' + pyenv_dir = f'{pymupdf_dir_abs}/pyenv-git' + os.environ['PYENV_ROOT'] = pyenv_dir + os.environ['PATH'] = f'{pyenv_dir}/bin:{os.environ["PATH"]}' + os.environ['PIPCL_GRAAL_PYTHON'] = sys.executable + + if venv >= 3: + shutil.rmtree(venv_name, ignore_errors=1) + if venv == 1 and os.path.exists(pyenv_dir) and os.path.exists(venv_name): + log(f'{venv=} and {venv_name=} already exists so not building pyenv or creating venv.') + else: + pipcl.git_get('https://github.com/pyenv/pyenv.git', pyenv_dir, branch='master') + run(f'cd {pyenv_dir} && src/configure && make -C src') + run(f'which pyenv') + run(f'pyenv install -v -s {graalpy}') + run(f'{pyenv_dir}/versions/{graalpy}/bin/graalpy -m venv {venv_name}') + e = run(f'. {venv_name}/bin/activate && python {shlex.join(sys.argv)}', + check=False, + ) + else: + venv_name = f'venv-pymupdf-{platform.python_version()}-{int.bit_length(sys.maxsize+1)}' + e = venv_run( + sys.argv, + venv_name, + recreate=(venv>=2), + clean=(venv>=3), + ) sys.exit(e) else: log(f'Warning, no commands specified so nothing to do.') + # Clone/update/build swig if specified. + swig_binary = pipcl.swig_get(swig, swig_quick) + if swig_binary: + os.environ['PYMUPDF_SETUP_SWIG'] = swig_binary + # Handle commands. # have_installed = False for command in commands: - + log(f'### {command=}.') if 0: pass @@ -540,14 +621,17 @@ def main(argv): elif command == 'cibw': # Build wheel(s) with cibuildwheel. if cibw_pyodide and env_extra.get('CIBW_BUILD') is None: - CIBW_BUILD = 'cp313*' + assert 0, f'Need a Python version for Pyodide.' + CIBW_BUILD = 'cp312*' env_extra['CIBW_BUILD'] = CIBW_BUILD log(f'Defaulting to {CIBW_BUILD=} for Pyodide.') + #if cibw_pyodide_version == None: + # cibw_pyodide_version = '0.28.0' cibuildwheel( env_extra, cibw_name or 'cibuildwheel', cibw_pyodide, - cibw_pyodide_version or '0.28.0', + cibw_pyodide_version, cibw_sdist, ) @@ -698,7 +782,7 @@ def cibuildwheel(env_extra, cibw_name, cibw_pyodide, cibw_pyodide_version, cibw_ log(f'{sdists=}') assert sdists - run(f'pip install --upgrade {cibw_name}') + run(f'pip install --upgrade --force-reinstall {cibw_name}') # Some general flags. if 'CIBW_BUILD_VERBOSITY' not in env_extra: @@ -1014,7 +1098,7 @@ def getmtime(path): pytest.main(args) return - if venv == 2: + if venv >= 2: run(f'pip install --upgrade {gh_release.test_packages}') else: log(f'{venv=}: Not installing test packages: {gh_release.test_packages}') @@ -1166,7 +1250,7 @@ def venv_in(path=None): return sys.prefix != sys.base_prefix -def venv_run(args, path, recreate=True): +def venv_run(args, path, recreate=True, clean=False): ''' Runs command inside venv and returns termination code. @@ -1180,7 +1264,13 @@ def venv_run(args, path, recreate=True): already exists. This avoids a delay in the common case where is already set up, but fails if exists but does not contain a valid venv. + clean: + If true we first delete . ''' + if clean: + log(f'Removing any existing venv {path}.') + assert path.startswith('venv-') + shutil.rmtree(path, ignore_errors=1) if recreate or not os.path.isdir(path): run(f'{sys.executable} -m venv {path}') if platform.system() == 'Windows': From 314609ac1526f29c2e663486aba671ce25ec8dd0 Mon Sep 17 00:00:00 2001 From: Julian Smith Date: Wed, 23 Jul 2025 11:00:54 +0100 Subject: [PATCH 05/10] setup.py: allow custom swig and clean mupdf builds, Also use pipcl.py's logging and run() directly. --- setup.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index f8356281a..1b8c7b295 100755 --- a/setup.py +++ b/setup.py @@ -176,6 +176,9 @@ Any other prefix is an error. + PYMUPDF_SETUP_SWIG + If set, we use this instead of `swig`. + WDEV_VS_YEAR If set, we use as Visual Studio year, for example '2019' or '2022'. @@ -204,22 +207,9 @@ import pipcl -_log_prefix = None -def log( text): - global _log_prefix - if not _log_prefix: - # This typically sets _log_prefix to `PyMuPDF/setup.py`. - p = os.path.abspath( __file__) - p, p1 = os.path.split( p) - p, p0 = os.path.split( p) - _log_prefix = os.path.join( p0, p1) - print(f'{_log_prefix}: {text}', file=sys.stdout) - sys.stdout.flush() - +log = pipcl.log0 -def run(command, check=1): - log(f'Running: {command}') - return subprocess.run( command, shell=1, check=check) +run = pipcl.run if 1: @@ -254,6 +244,7 @@ def run(command, check=1): PYMUPDF_SETUP_DUMMY = os.environ.get('PYMUPDF_SETUP_DUMMY') log(f'{PYMUPDF_SETUP_DUMMY=}') +PYMUPDF_SETUP_SWIG = os.environ.get('PYMUPDF_SETUP_SWIG') def _fs_remove(path): ''' @@ -478,7 +469,7 @@ def get_mupdf_internal(out, location=None, sha=None, local_tgz=None): local_dir = 'mupdf-git' # Try to update existing checkout. - e = run(f'cd {local_dir} && git pull && git submodule update --init', check=False).returncode + e = run(f'cd {local_dir} && git pull && git submodule update --init', check=False) if e: # No existing git checkout, so do a fresh clone. _fs_remove(local_dir) @@ -598,6 +589,8 @@ def build(): ''' pipcl.py `build_fn()` callback. ''' + #pipcl.show_sysconfig() + if PYMUPDF_SETUP_DUMMY == '1': log(f'{PYMUPDF_SETUP_DUMMY=} Building dummy wheel with no files.') return list() @@ -642,6 +635,7 @@ def build(): g_py_limited_api, PYMUPDF_SETUP_MUPDF_REFCHECK_IF, PYMUPDF_SETUP_MUPDF_TRACE_IF, + PYMUPDF_SETUP_SWIG, ) log( f'build(): mupdf_build_dir={mupdf_build_dir!r}') @@ -884,6 +878,7 @@ def build_mupdf_unix( g_py_limited_api, PYMUPDF_SETUP_MUPDF_REFCHECK_IF, PYMUPDF_SETUP_MUPDF_TRACE_IF, + PYMUPDF_SETUP_SWIG, ): ''' Builds MuPDF. @@ -997,13 +992,20 @@ def build_mupdf_unix( if g_py_limited_api: build_prefix += f'Py_LIMITED_API_{pipcl.current_py_limited_api()}-' unix_build_dir = f'{mupdf_local}/build/{build_prefix}{build_type}' + PYMUPDF_SETUP_MUPDF_CLEAN = os.environ.get('PYMUPDF_SETUP_MUPDF_CLEAN') + if PYMUPDF_SETUP_MUPDF_CLEAN == '1': + log(f'{PYMUPDF_SETUP_MUPDF_CLEAN=}, deleting {unix_build_dir=}.') + shutil.rmtree(unix_build_dir, ignore_errors=1) # We need MuPDF's Python bindings, so we build MuPDF with # `mupdf/scripts/mupdfwrap.py` instead of running `make`. # command = f'cd {mupdf_local} &&' for n, v in env.items(): command += f' {n}={shlex.quote(v)}' - command += f' {sys.executable} ./scripts/mupdfwrap.py -d build/{build_prefix}{build_type} -b' + command += f' {sys.executable} ./scripts/mupdfwrap.py' + if PYMUPDF_SETUP_SWIG: + command += f' --swig {shlex.quote(PYMUPDF_SETUP_SWIG)}' + command += f' -d build/{build_prefix}{build_type} -b' #command += f' --m-target libs' if PYMUPDF_SETUP_MUPDF_REFCHECK_IF: command += f' --refcheck-if "{PYMUPDF_SETUP_MUPDF_REFCHECK_IF}"' @@ -1103,6 +1105,7 @@ def _build_extension( mupdf_local, mupdf_build_dir, build_type, g_py_limited_api prerequisites_compile = f'{mupdf_local}/include', prerequisites_link = libraries, py_limited_api = g_py_limited_api, + swig = PYMUPDF_SETUP_SWIG, ) return path_so_leaf From 460313907ad0bdbcd97fec6ded0cc3f3e1472379 Mon Sep 17 00:00:00 2001 From: Julian Smith Date: Thu, 24 Jul 2025 18:33:34 +0100 Subject: [PATCH 06/10] tests/: allow skipping of tests until a named test. --- tests/README.md | 8 ++++++++ tests/conftest.py | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index 96706f6c0..b793d2489 100644 --- a/tests/README.md +++ b/tests/README.md @@ -71,3 +71,11 @@ having been built with PyMuPDF's customized configuration, ``fitz/_config.h``. One can skip this particular test by adding ``-k 'not test_textbox3'`` to the pytest command line. + + +## Resuming at a particular test. + +To skip tests before a particular test, set PYMUPDF_PYTEST_RESUME to the name +of the function. + +For example PYMUPDF_PYTEST_RESUME=test_haslinks. diff --git a/tests/conftest.py b/tests/conftest.py index 4017de580..4c6f34770 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,8 +7,10 @@ import pytest +PYMUPDF_PYTEST_RESUME = os.environ.get('PYMUPDF_PYTEST_RESUME') + @pytest.fixture(autouse=True) -def wrap(*args, **kwargs): +def wrap(request): ''' Check that tests return with empty MuPDF warnings buffer. For example this detects failure to call fz_close_output() before fz_drop_output(), which @@ -17,6 +19,16 @@ def wrap(*args, **kwargs): As of 2024-09-12 we also detect whether tests leave fds open; but for now do not fail tests, because many tests need fixing. ''' + global PYMUPDF_PYTEST_RESUME + if PYMUPDF_PYTEST_RESUME: + # Skip all tests until we reach a matching name. + if PYMUPDF_PYTEST_RESUME == request.function.__name__: + print(f'### {PYMUPDF_PYTEST_RESUME=}: resuming at {request.function.__name__=}.') + PYMUPDF_PYTEST_RESUME = None + else: + print(f'### {PYMUPDF_PYTEST_RESUME=}: Skipping {request.function.__name__=}.') + return + wt = pymupdf.TOOLS.mupdf_warnings() assert not wt, f'{wt=}' assert not pymupdf.TOOLS.set_small_glyph_heights() @@ -99,7 +111,7 @@ def get_members(a): next_fd_after = os.open(__file__, os.O_RDONLY) os.close(next_fd_after) if next_fd_after != next_fd_before: - print(f'Test has leaked fds, {next_fd_before=} {next_fd_after=}. {args=} {kwargs=}.') + print(f'Test has leaked fds, {next_fd_before=} {next_fd_after=}.') #assert 0, f'Test has leaked fds, {next_fd_before=} {next_fd_after=}. {args=} {kwargs=}.' if 0: From f7932c335d1e659b042a29182fde9091c1b4517a Mon Sep 17 00:00:00 2001 From: Julian Smith Date: Thu, 24 Jul 2025 18:34:16 +0100 Subject: [PATCH 07/10] tests/: some exclusions of slow tests on graal. tests/test_pixmap.py: test_2093: exclude on graal because slow. tests/conftest.py: don't show or check fds on graal because lots of failures. tests/test_tables.py: test_2979() is slow on graal. test_add_lines() seems to break tests/conftest.py's resetting and checking of global state for each test. For example after test_add_lines(), pymupdf.TOOLS.set_small_glyph_heights() always seeme to return true, so all later tests are marked as ERROR. --- tests/conftest.py | 17 +++++++++++++---- tests/test_general.py | 7 +++++++ tests/test_memory.py | 6 ++++++ tests/test_pixmap.py | 3 +++ tests/test_tables.py | 9 +++++++++ 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4c6f34770..e1d02297a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,11 +31,19 @@ def wrap(request): wt = pymupdf.TOOLS.mupdf_warnings() assert not wt, f'{wt=}' - assert not pymupdf.TOOLS.set_small_glyph_heights() + if platform.python_implementation() == 'GraalVM': + pymupdf.TOOLS.set_small_glyph_heights() + else: + assert not pymupdf.TOOLS.set_small_glyph_heights() next_fd_before = os.open(__file__, os.O_RDONLY) os.close(next_fd_before) - if platform.system() == 'Linux': + if platform.system() == 'Linux' and platform.python_implementation() != 'GraalVM': + test_fds = True + else: + test_fds = False + + if test_fds: # Gather detailed information about leaked fds. def get_fds(): import subprocess @@ -94,7 +102,7 @@ def get_members(a): assert pymupdf.JM_annot_id_stem == JM_annot_id_stem, \ f'pymupdf.JM_annot_id_stem has changed from {JM_annot_id_stem!r} to {pymupdf.JM_annot_id_stem!r}' - if platform.system() == 'Linux': + if test_fds: # Show detailed information about leaked fds. open_fds_after, open_fds_after_l = get_fds() if open_fds_after != open_fds_before: @@ -110,7 +118,8 @@ def get_members(a): next_fd_after = os.open(__file__, os.O_RDONLY) os.close(next_fd_after) - if next_fd_after != next_fd_before: + + if test_fds and next_fd_after != next_fd_before: print(f'Test has leaked fds, {next_fd_before=} {next_fd_after=}.') #assert 0, f'Test has leaked fds, {next_fd_before=} {next_fd_after=}. {args=} {kwargs=}.' diff --git a/tests/test_general.py b/tests/test_general.py index 16c6955ed..aa3e53dbd 100644 --- a/tests/test_general.py +++ b/tests/test_general.py @@ -438,6 +438,10 @@ def test_2238(): def test_2093(): + if platform.python_implementation() == 'GraalVM': + print(f'test_2093(): Not running because slow on GraalVM.') + return + doc = pymupdf.open(f'{scriptdir}/resources/test2093.pdf') def average_color(page): @@ -597,6 +601,9 @@ def test_2692(): def test_2596(): """Confirm correctly abandoning cache when reloading a page.""" + if platform.python_implementation() == 'GraalVM': + print(f'test_2596(): not running on Graal.') + return doc = pymupdf.Document(f"{scriptdir}/resources/test_2596.pdf") page = doc[0] pix0 = page.get_pixmap() # render the page diff --git a/tests/test_memory.py b/tests/test_memory.py index 48a3a1bd2..3985a67e3 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -41,8 +41,14 @@ def get_stat(): def get_stat(): return 0 n = 1000 + verbose = False + if platform.python_implementation() == 'GraalVM': + n = 10 + verbose = True stats = [1] * n for i in range(n): + if verbose: + print(f'{i+1}/{n}.', flush=1) root = os.path.abspath(f'{__file__}/../../tests/resources') with open(f'{root}/test_2791_content.pdf', 'rb') as content_pdf: with open(f'{root}/test_2791_coverpage.pdf', 'rb') as coverpage_pdf: diff --git a/tests/test_pixmap.py b/tests/test_pixmap.py index 08556363a..c8f89dbd0 100644 --- a/tests/test_pixmap.py +++ b/tests/test_pixmap.py @@ -375,6 +375,9 @@ def test_3848(): # Takes 40m on Github. print(f'test_3848(): not running on valgrind because very slow.', flush=1) return + if platform.python_implementation() == 'GraalVM': + print(f'test_3848(): Not running because slow on GraalVM.') + return path = os.path.normpath(f'{__file__}/../../tests/resources/test_3848.pdf') with pymupdf.open(path) as document: for i in range(len(document)): diff --git a/tests/test_tables.py b/tests/test_tables.py index 3ad46d158..2c537de52 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -3,6 +3,7 @@ from pprint import pprint import textwrap import pickle +import platform import pymupdf @@ -196,6 +197,10 @@ def test_3062(): """Tests the fix for #3062. After table extraction, a rotated page should behave and look like as before.""" + if platform.python_implementation() == 'GraalVM': + print(f'test_3062(): Not running because slow on GraalVM.') + return + filename = os.path.join(scriptdir, "resources", "test_3062.pdf") doc = pymupdf.open(filename) page = doc[0] @@ -223,6 +228,10 @@ def test_strict_lines(): def test_add_lines(): """Test new parameter add_lines for table recognition.""" + if platform.python_implementation() == 'GraalVM': + print(f'test_add_lines(): Not running because breaks later tests on GraalVM.') + return + filename = os.path.join(scriptdir, "resources", "small-table.pdf") doc = pymupdf.open(filename) page = doc[0] From 30871fb1c7780c373e9337883493f2e73fd3d8c9 Mon Sep 17 00:00:00 2001 From: Julian Smith Date: Mon, 4 Aug 2025 15:39:55 +0100 Subject: [PATCH 08/10] tests/: more concise specifications of large expected warnings text. --- tests/test_2548.py | 3 ++- tests/test_textextract.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_2548.py b/tests/test_2548.py index 2d354fdb0..f3a2b2db5 100644 --- a/tests/test_2548.py +++ b/tests/test_2548.py @@ -33,7 +33,8 @@ def test_2548(): # versions with updated MuPDF also fix the bug. rebased = hasattr(pymupdf, 'mupdf') if pymupdf.mupdf_version_tuple >= (1, 27): - expected = 'format error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing' + expected = 'format error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing' + expected = '\n'.join([expected] * 5) else: expected = 'format error: cycle in structure tree\nstructure tree broken, assume tree is missing' if rebased: diff --git a/tests/test_textextract.py b/tests/test_textextract.py index 46b137a37..488ddd76c 100644 --- a/tests/test_textextract.py +++ b/tests/test_textextract.py @@ -374,12 +374,13 @@ def get_all_page_from_pdf(document, last_page=None): assert texts1 == texts0 + wt = pymupdf.TOOLS.mupdf_warnings() if pymupdf.mupdf_version_tuple < (1, 27): - wt = pymupdf.TOOLS.mupdf_warnings() assert wt == 'Actualtext with no position. Text may be lost or mispositioned.\n... repeated 434 times...' else: - wt = pymupdf.TOOLS.mupdf_warnings() - assert wt == 'format error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing\nformat error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing' + expected = 'format error: No common ancestor in structure tree\nstructure tree broken, assume tree is missing' + expected = '\n'.join([expected] * 56) + assert wt == expected def test_3650(): path = os.path.normpath(f'{__file__}/../../tests/resources/test_3650.pdf') From 85af0972f61b1532de0bc26b6d2831983f094171 Mon Sep 17 00:00:00 2001 From: Julian Smith Date: Wed, 23 Jul 2025 12:47:51 +0100 Subject: [PATCH 09/10] tests/run_compound.py: graal breaks printing of platform.uname(). --- tests/run_compound.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/run_compound.py b/tests/run_compound.py index 2d204072a..927b721e3 100755 --- a/tests/run_compound.py +++ b/tests/run_compound.py @@ -89,7 +89,8 @@ def main(): log(f'{platform.platform()=}') log(f'{platform.python_version()=}') log(f'{platform.system()=}') - log(f'{platform.uname()=}') + if sys.implementation.name != 'graalpy': + log(f'{platform.uname()=}') log(f'{sys.executable=}') log(f'{sys.version=}') log(f'{sys.version_info=}') From 99931157da592c1331d5658f4314b97b44bd1146 Mon Sep 17 00:00:00 2001 From: Julian Smith Date: Tue, 5 Aug 2025 18:07:33 +0100 Subject: [PATCH 10/10] .github/workflows/test_quick.yml: test with mupdf master and release branch. --- .github/workflows/test_quick.yml | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test_quick.yml b/.github/workflows/test_quick.yml index 422ecfc74..e07aadf35 100644 --- a/.github/workflows/test_quick.yml +++ b/.github/workflows/test_quick.yml @@ -13,23 +13,34 @@ on: jobs: - test_quick: - name: Test quick + master: + name: mupdf master runs-on: ${{ matrix.os }} strategy: matrix: - # We test on just Ubuntu with MuPDF master. - # os: [ubuntu-latest] - - # Avoid cancelling of all cibuildwheel runs after a single failure. fail-fast: false + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - name: mupdf master + env: + PYMUPDF_test_args: ${{inputs.args}} + run: + python scripts/test.py build test -m 'git:--branch master https://github.com/ArtifexSoftware/mupdf.git' -a PYMUPDF_test_args + release: + name: mupdf release + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + fail-fast: false steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 - - name: test_quick + - name: mupdf release env: PYMUPDF_test_args: ${{inputs.args}} run: - python scripts/test.py build test -a PYMUPDF_test_args + python scripts/test.py build test -m 'git:--branch 1.26.x https://github.com/ArtifexSoftware/mupdf.git' -a PYMUPDF_test_args