diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..9815f9326 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,62 @@ +# Run scripts/test.py directly on multiple Github servers. Instead of +# specifying individual inputs, we support a single string input which is used +# for the command line directly. +# +# This ensures we behave exactly like scripts/test.py, without confusion caused +# by having to translate between differing APIs. + +name: Tests + +on: + #schedule: + # - cron: '47 4 * * *' + #pull_request: + # branches: [main] + workflow_dispatch: + inputs: + args: + type: string + default: '' + description: 'Arguments to pass to scripts/test.py' + +jobs: + + test: + name: Test + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-2019, macos-13, macos-14] + + # Avoid cancelling of all runs after a single failure. + fail-fast: false + + steps: + + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + # https://github.com/pypa/cibuildwheel/issues/2114 + # https://cibuildwheel.pypa.io/en/stable/faq/#emulation + # + - name: Set up QEMU + if: runner.os == 'Linux' && runner.arch == 'X64' + uses: docker/setup-qemu-action@v3 + with: + platforms: all + + - name: test + env: + PYMUPDF_test_args: ${{inputs.args}} + run: + python scripts/test.py -a PYMUPDF_test_args + + # Upload generated wheels, to be accessible from github Actions page. + # + - uses: actions/upload-artifact@v4 + with: + path: ./wheelhouse/pymupdf*.whl + name: artifact-${{ matrix.os }} diff --git a/changes.txt b/changes.txt index ae98fa448..febe272d8 100644 --- a/changes.txt +++ b/changes.txt @@ -32,7 +32,9 @@ Change Log * Added runtime assert that that PyMuPDF and MuPDF were built with compatible NDEBUG settings (related to `4390 `_). * Simplified handling of filename/filetype when opening documents. - + * Removed PDF linearization support. + * Calls to `Document.save()` with `linear` set to true will now raise an exception. + * See https://artifex.com/blog/mupdf-removes-linearisation for more information. **Changes in version 1.25.5 (2025-03-31)** diff --git a/pipcl.py b/pipcl.py index 8557f1500..4a3edd0e1 100644 --- a/pipcl.py +++ b/pipcl.py @@ -21,6 +21,7 @@ import os import platform import re +import shlex import shutil import site import subprocess @@ -654,12 +655,12 @@ def add_str(content, to_): z.writestr(f'{dist_info_dir}/RECORD', record.get(f'{dist_info_dir}/RECORD')) st = os.stat(path) - log1( f'Have created wheel size={st.st_size}: {path}') + log1( f'Have created wheel size={st.st_size:,}: {path}') if g_verbose >= 2: with zipfile.ZipFile(path, compression=self.wheel_compression) as z: log2(f'Contents are:') for zi in sorted(z.infolist(), key=lambda z: z.filename): - log2(f' {zi.file_size: 10d} {zi.filename}') + log2(f' {zi.file_size: 10,d} {zi.filename}') return os.path.basename(path) @@ -1901,6 +1902,105 @@ def git_items( directory, submodules=False): return ret +def git_get( + remote, + local, + *, + branch=None, + depth=1, + env_extra=None, + tag=None, + update=True, + submodules=True, + ): + ''' + Ensures that is a git checkout (at either , or HEAD) + of a remote repository. + + Exactly one of and must be specified. + + Args: + remote: + Remote git repostitory, for example + 'https://github.com/ArtifexSoftware/mupdf.git'. + local: + Local directory. If /.git exists, we attempt to run `git + update` in it. + branch: + Branch to use. + depth: + Depth of local checkout when cloning and fetching, or None. + env_extra: + Dict of extra name=value environment variables to use whenever we + run git. + tag: + Tag to use. + update: + If false we do not update existing repository. Might be useful if + testing without network access. + submodules: + If true, we clone with `--recursive --shallow-submodules` and run + `git submodule update --init --recursive` before returning. + ''' + log0(f'{remote=} {local=} {branch=} {tag=}') + assert (branch and not tag) or (not branch and tag), f'Must specify exactly one of and .' + + depth_arg = f' --depth {depth}' if depth else '' + + def do_update(): + # This seems to pull in the entire repository. + log0(f'do_update(): attempting to update {local=}.') + # Remove any local changes. + run(f'cd {local} && git checkout .', env_extra=env_extra) + if tag: + # `-u` avoids `fatal: Refusing to fetch into current branch`. + # Using '+' and `revs/tags/` prefix seems to avoid errors like: + # error: cannot update ref 'refs/heads/v3.16.44': + # trying to write non-commit object + # 06c4ae5fe39a03b37a25a8b95214d9f8f8a867b8 to branch + # 'refs/heads/v3.16.44' + # + run(f'cd {local} && git fetch -fuv{depth_arg} {remote} +refs/tags/{tag}:refs/tags/{tag}', env_extra=env_extra) + run(f'cd {local} && git checkout {tag}', env_extra=env_extra) + if branch: + # `-u` avoids `fatal: Refusing to fetch into current branch`. + run(f'cd {local} && git fetch -fuv{depth_arg} {remote} {branch}:{branch}', env_extra=env_extra) + run(f'cd {local} && git checkout {branch}', env_extra=env_extra) + + do_clone = True + if os.path.isdir(f'{local}/.git'): + if update: + # Try to update existing checkout. + try: + do_update() + do_clone = False + except Exception as e: + log0(f'Failed to update existing checkout {local}: {e}') + else: + do_clone = False + + if do_clone: + # No existing git checkout, so do a fresh clone. + #_fs_remove(local) + log0(f'Cloning to: {local}') + command = f'git clone --config core.longpaths=true{depth_arg}' + if submodules: + command += f' --recursive --shallow-submodules' + if branch: + command += f' -b {branch}' + if tag: + command += f' -b {tag}' + command += f' {remote} {local}' + run(command, env_extra=env_extra) + do_update() + + if submodules: + run(f'cd {local} && git submodule update --init --recursive', env_extra=env_extra) + + # Show sha of checkout. + run( f'cd {local} && git show --pretty=oneline|head -n 1', check=False) + + def run( command, *, @@ -1951,9 +2051,14 @@ def run( env = os.environ.copy() env.update(env_extra) lines = _command_lines( command) - nl = '\n' if verbose: - log1( f'Running: {nl.join(lines)}', caller=caller+1) + text = f'Running:' + if env_extra: + for k in sorted(env_extra.keys()): + text += f' {k}={shlex.quote(env_extra[k])}' + nl = '\n' + text += f' {nl.join(lines)}' + log1(text, caller=caller+1) sep = ' ' if windows() else ' \\\n' command2 = sep.join( lines) cp = subprocess.run( @@ -1990,6 +2095,39 @@ def linux(): def openbsd(): return platform.system() == 'OpenBSD' + +def show_system(): + ''' + Show useful information about the system plus argv and environ. + ''' + def log(text): + log0(text, caller=3) + + #log(f'{__file__=}') + #log(f'{__name__=}') + log(f'{os.getcwd()=}') + log(f'{platform.machine()=}') + log(f'{platform.platform()=}') + log(f'{platform.python_version()=}') + log(f'{platform.system()=}') + log(f'{platform.uname()=}') + log(f'{sys.executable=}') + log(f'{sys.version=}') + log(f'{sys.version_info=}') + log(f'{list(sys.version_info)=}') + + log(f'CPU bits: {cpu_bits()}') + + log(f'sys.argv ({len(sys.argv)}):') + for i, arg in enumerate(sys.argv): + log(f' {i}: {arg!r}') + + log(f'os.environ ({len(os.environ)}):') + for k in sorted( os.environ.keys()): + v = os.environ[ k] + log( f' {k}: {v!r}') + + class PythonFlags: ''' Compile/link flags for the current python, for example the include path @@ -2164,6 +2302,10 @@ def _command_lines( command): return lines +def cpu_bits(): + return int.bit_length(sys.maxsize+1) + + def _cpu_name(): ''' Returns `x32` or `x64` depending on Python build. @@ -2418,7 +2560,7 @@ def log2(text='', caller=1): def _log(text, level, caller): ''' - Logs lines with prefix. + Logs lines with prefix, if is lower than . ''' if level <= g_verbose: fr = inspect.stack(context=0)[caller] @@ -2445,49 +2587,6 @@ def relpath(path, start=None): return os.path.relpath(path, start) -def number_sep( s): - ''' - Simple number formatter, adds commas in-between thousands. `s` can be a - number or a string. Returns a string. - - >>> number_sep(1) - '1' - >>> number_sep(12) - '12' - >>> number_sep(123) - '123' - >>> number_sep(1234) - '1,234' - >>> number_sep(12345) - '12,345' - >>> number_sep(123456) - '123,456' - >>> number_sep(1234567) - '1,234,567' - >>> number_sep(-131072) - '-131,072' - ''' - if not isinstance( s, str): - s = str( s) - ret = '' - if s.startswith('-'): - ret += '-' - s = s[1:] - c = s.find( '.') - if c==-1: c = len(s) - end = s.find('e') - if end == -1: end = s.find('E') - if end == -1: end = len(s) - for i in range( end): - ret += s[i] - if ic and i. + self.glob_pattern = glob_pattern + self.items0 = self._items() + def get(self): + ''' + Returns list of new matches of - paths of files that + were not present previously, or have different mtimes or have different + contents. + ''' + ret = list() + items = self._items() + for path, id_ in items.items(): + id0 = self.items0.get(path) + if id0 != id_: + #mtime0, hash0 = id0 + #mtime1, hash1 = id_ + #log0(f'New/modified file {path=}.') + #log0(f' {mtime0=} {"==" if mtime0==mtime1 else "!="} {mtime1=}.') + #log0(f' {hash0=} {"==" if hash0==hash1 else "!="} {hash1=}.') + ret.append(path) + return ret + def get_one(self): + ''' + Returns new match of , asserting that there is exactly + one. + ''' + ret = self.get() + assert len(ret) == 1, f'{len(ret)=}' + return ret[0] + def _file_id(self, path): + mtime = os.stat(path).st_mtime + with open(path, 'rb') as f: + hash_ = hashlib.file_digest(f, hashlib.md5).digest() + return mtime, hash_ + def _items(self): + ret = dict() + for path in glob.glob(self.glob_pattern): + if os.path.isfile(path): + ret[path] = self._file_id(path) + return ret diff --git a/scripts/test.py b/scripts/test.py index 7c425c867..973a404b5 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -4,70 +4,154 @@ Examples: - ./PyMuPDF/scripts/test.py --mupdf mupdf buildtest + ./PyMuPDF/scripts/test.py --m mupdf build test Build and test with pre-existing local mupdf/ checkout. - ./PyMuPDF/scripts/test.py buildtest + ./PyMuPDF/scripts/test.py build test Build and test with default internal download of mupdf. - ./PyMuPDF/scripts/test.py --mupdf 'git:https://git.ghostscript.com/mupdf.git' buildtest - Build and test with internal checkout of mupdf master. + ./PyMuPDF/scripts/test.py -m 'git:https://git.ghostscript.com/mupdf.git' build test + Build and test with internal checkout of MuPDF master. - ./PyMuPDF/scripts/test.py --mupdf 'git:--branch 1.26.x https://github.com/ArtifexSoftware/mupdf.git' buildtest - Build and test using internal checkout of mupdf 1.26.x branch from Github. + ./PyMuPDF/scripts/test.py -m 'git:--branch 1.26.x https://github.com/ArtifexSoftware/mupdf.git' build test + Build and test using internal checkout of mupdf 1.26.x branch from + Github. Usage: - scripts/test.py -* Commands are handled in order, so for example `build` should usually be - before `test`. +* Command line arguments are called parameters if they start with `-`, + otherwise they are called commands. +* Parameters are evaluated first in the order that they were specified. +* Then commands are run in the order in which they were specified. +* Usually command `test` would be specified after a `build`, `install` or + `wheel` command. +* Parameters and commands can be interleaved but it may be clearer to separate + them on the command line. + +Other: * If we are not already running inside a Python venv, we automatically create a venv and re-run ourselves inside it. - -* We build directly with pip (unlike gh_release.py, which builds with - cibuildwheel). - +* Build/wheel/install commands always install into the venv. +* Tests use whatever PyMuPDF/MuPDF is currently installed in the venv. * We run tests with pytest. * One can generate call traces by setting environment variables in debug builds. For details see: https://mupdf.readthedocs.io/en/latest/language-bindings.html#environmental-variables -Options: - --help - -h - Show help. +Command line args: + + -a + Read next space-separated argument(s) from environmental variable + . + * Does nothing if is unset. + * Useful when running via Github action. + -b - Set build type for `build` or `buildtest` commands. `` should - be one of 'release', 'debug', 'memento'. [This makes `build` set - environment variable `PYMUPDF_SETUP_MUPDF_BUILD_TYPE`, which is used by - PyMuPDF's `setup.py`.] + Set build type for `build` commands. `` should be one of + 'release', 'debug', 'memento'. [This makes `build` set environment + variable `PYMUPDF_SETUP_MUPDF_BUILD_TYPE`, which is used by PyMuPDF's + `setup.py`.] + + --build-flavour + Combination of 'p', 'b', 'd'. See ../setup.py's description of + PYMUPDF_SETUP_FLAVOUR. Default is 'pbd', i.e. self-contained PyMuPDF + wheels including MuPDF build-time files. + + --build-isolation 0|1 + If true (the default on non-OpenBSD systems), we let pip create and use + its own new venv to build PyMuPDF. Otherwise we force pip to use the + current venv. + + --cibw-name + Name to use when installing cibuildwheel, e.g.: + --cibw-name cibuildwheel==3.0.0b1 + Default is `cibuildwheel`, i.e. the current release. + + --cibw-pyodide 0|1 + Experimental, make `cibuild` command build a pyodide wheel. + 2025-05-27: this fails when building mupdf C API - `ld -r -b binary + ...` fails with: + emcc: error: binary: No such file or directory ("binary" was expected to be an input file, based on the commandline arguments provided) + -d Equivalent to `-b debug`. + + -e = + Add to environment used in build and test commands. Can be specified + multiple times. + -f 0|1 If 1 we also test alias `fitz` as well as `pymupdf`. Default is '0'. + + --gdb 0|1 + Run tests under gdb. Requires user interaction. + + --help + -h + Show help. -i Set PyMuPDF implementations to test. must contain only these individual characters: 'r' - rebased. 'R' - rebased without optimisations. Default is 'r'. Also see `PyMuPDF:tests/run_compound.py`. + -k Specify which test(s) to run; passed straight through to pytest's `-k`. For example `-k test_3354`. + -m | --mupdf Location of local mupdf/ directory or 'git:...' to be used when building PyMuPDF. [This sets environment variable PYMUPDF_SETUP_MUPDF_BUILD, which is used by PyMuPDF/setup.py. If not specified PyMuPDF will download its default mupdf .tgz.] + + -M 0|1 + + --build-mupdf 0|1 + Whether to rebuild mupdf when we build PyMuPDF. Default is 1. + + -o + Control whether we do nothing on the current platform. + * is a comma-separated list of names. + * If is empty (the default), we always run normally. + * Otherwise we only run if an item in matches (case + insensitive) platform.system(). + * For example `-o linux,darwin` will do nothing unless on Linux or + MacOS. + -p Set pytest options; default is ''. + -P 0|1 - If 1, automatically install required packages such as Valgrind. Default - is 0. + If 1, automatically install required system packages such as + Valgrind. Default is 0. + + --pybind 0|1 + Experimental, for investigating + https://github.com/pymupdf/PyMuPDF/issues/3869. Runs run basic code + inside C++ pybind. Requires `sudo apt install pybind11-dev` or similar. + + --pyodide-build-version + Version of Python package pyodide-build; if None (the default) we use + latest available version. + 2025-02-13: pyodide_build_version='0.29.3' works. + -s 0 | 1 If 1 (the default), build with Python Limited API/Stable ABI. + [This simply sSets $PYMUPDF_SETUP_PY_LIMITED_API, which is used by + PyMuPDF/setup.py.] + + --sync-paths + Do not run anything, instead write required files/directories/checkouts + to stdout, one per line. This is to help with automated running on + remote machines. + + --system-site-packages 0|1 + If 1, use `--system-site-packages` when creating venv. Defaults is 0. + -t Pytest test names, comma-separated. Should be relative to PyMuPDF directory. For example: @@ -75,56 +159,55 @@ -t tests/test_general.py::test_subset_fonts To specify multiple tests, use comma-separated list and/or multiple `-t ` args. + + --timeout + Sets timeout when running tests. + + -T | --pytest-prefix + Use specified prefix when running pytest. E.g. `gdb --args`. + -v 0|1|2 0 - do not use a venv. 1 - Use venv. If it already exists, we assume the existing directory was created by us earlier and is a valid venv containing all necessary packages; this saves a little time. - 2 - use venv. + 2 - Use venv The default is 2. - --build-isolation 0|1 - If true (the default on non-OpenBSD systems), we let pip create and use - its own new venv to build PyMuPDF. Otherwise we force pip to use the - current venv. - --build-flavour - Combination of 'p', 'b', 'd'. See ../setup.py's description of - PYMUPDF_SETUP_FLAVOUR. Default is 'pbd', i.e. self-contained PyMuPDF - wheels including MuPDF build-time files. - --build-mupdf 0|1 - Whether to rebuild mupdf when we build PyMuPDF. Default is 1. - --gdb 0|1 - Run tests under gdb. Requires user interaction. - --pytest-prefix - Use specified prefix when running pytest. E.g. `gdb --args`. - --pybind 0|1 - Experimental, for investigating - https://github.com/pymupdf/PyMuPDF/issues/3869. Runs run basic code - inside C++ pybind. Requires `sudo apt install pybind11-dev` or similar. - --pymupdf-pypi - Do not build PyMuPDF, instead install with `pip install `. For - example allows testing of a specific version with `--pymupdf-pypi - pymupdf==x.y.z`. - --system-site-packages 0|1 - If 1, use `--system-site-packages` when creating venv. Defaults is 0. - --timeout - Sets timeout when running tests. + --valgrind 0|1 Use valgrind in `test` or `buildtest`. This will run `sudo apt update` and `sudo apt install valgrind`. - --valgrind-args - Extra args to valgrind. Commands: + build Builds and installs PyMuPDF into venv, using `pip install .../PyMuPDF`. + buildtest Same as 'build test'. - test - Runs PyMuPDF's pytest tests in venv. Default is to test rebased and - unoptimised rebased; use `-i` to change this. - wheel - Build wheel. - pyodide_wheel + + cibw + Build and test PyMuPDF wheel(s) using cibuildwheel. Wheels are placed + in directory `wheelhouse`. + * We do not attempt to install wheels. + * So it is generally not useful to do `cibw test`. + + If CIBW_BUILD is unset, we set it as follows: + * On Github we build and test all supported Python versions. + * Otherwise we build and test the current Python version only. + + If CIBW_ARCHS is unset we set $CIBW_ARCHS_WINDOWS, $CIBW_ARCHS_MACOS + and $CIBW_ARCHS_LINUX to auto64 if they are unset. + + Additionally, if running on Github ($GITHUB_ACTIONS=true) and + $CIBW_ARCHS_LINUX is unset, we set $CIBW_ARCHS_LINUX to 'auto64 + aarch64' so that we build for aarch64 using emulation. This is required + as of 2025-05-23 because there is no native aarch64 host available. + + install + Install with `pip install --force-reinstall `. + + pyodide Build Pyodide wheel. We clone `emsdk.git`, set it up, and run `pyodide build`. This runs our setup.py with CC etc set up to create Pyodide binaries in a wheel called, for example, @@ -133,6 +216,13 @@ It seems that sys.version must match the Python version inside emsdk; as of 2025-02-14 this is 3.12. Otherwise we get build errors such as: [wasm-validator error in function 723] unexpected false: all used features should be allowed, on ... + + test + Runs PyMuPDF's pytest tests. Default is to test rebased and unoptimised + rebased; use `-i` to change this. + + wheel + Build and install wheel. Environment: @@ -152,11 +242,15 @@ import textwrap -pymupdf_dir = os.path.abspath( f'{__file__}/../..') +pymupdf_dir_abs = os.path.abspath( f'{__file__}/../..') + +try: + sys.path.insert(0, pymupdf_dir_abs) + import pipcl +finally: + del sys.path[0] -sys.path.insert(0, pymupdf_dir) -import pipcl -del sys.path[0] +pymupdf_dir = pipcl.relpath(pymupdf_dir_abs) log = pipcl.log0 run = pipcl.run @@ -171,35 +265,35 @@ def main(argv): show_help() return - log(f'{sys.executable=}') - log(f'{sys.version=}') - build_isolation = None - valgrind = False - valgrind_args = '' - s = True - build_do = 'i' - build_type = None - build_mupdf = True - build_flavour = 'pbd' - gdb = False - test_fitz = False + cibw_name = 'cibuildwheel' + cibw_pyodide = None + commands = list() + env_extra = dict() implementations = 'r' - test_names = list() - venv = 2 - pytest_prefix = None + mupdf_sync = None + os_names = list() + system_packages = False pybind = False - pytest_options = None - timeout = None - pytest_k = None - system_site_packages = False pyodide_build_version = None - packages = False - pymupdf_pypi = None + pytest_options = '' + pytest_prefix = None + show_help = False + sync_paths = False + system_site_packages = False + test_fitz = False + test_names = list() + test_timeout = None + valgrind = False + warnings = list() + venv = 2 options = os.environ.get('PYMUDF_SCRIPTS_TEST_options', '') options = shlex.split(options) + # Parse args and update the above state. We do this before moving into a + # venv, partly so we can return errors immediately. + # args = iter(options + argv[1:]) i = 0 while 1: @@ -208,138 +302,208 @@ def main(argv): except StopIteration: arg = None break - if not arg.startswith('-'): - break + + if 0: + pass + + elif arg == '-a': + _name = next(args) + _value = os.environ.get(_name, '') + _args = shlex.split(_value) + list(args) + args = iter(_args) + elif arg == '-b': - build_type = next(args) + env_extra['PYMUPDF_SETUP_MUPDF_BUILD_TYPE'] = next(args) + + elif arg == '--build-flavour': + env_extra['PYMUPDF_SETUP_FLAVOUR'] = next(args) + elif arg == '--build-isolation': build_isolation = int(next(args)) + + elif arg == '--cibw-name': + cibw_name = next(args) + + elif arg == '--cibw-pyodide': + cibw_pyodide = next(args) + elif arg == '-d': - build_type = 'debug' + env_extra['PYMUPDF_SETUP_MUPDF_BUILD_TYPE'] = 'debug' + + elif arg == '-e': + _nv = next(args) + assert '=' in _nv, f'-e = does not contain "=": {_nv!r}' + _name, _value = _nv.split('=', 1) + env_extra[_name] = _value + elif arg == '-f': test_fitz = int(next(args)) + + elif arg == '--gdb': + _gdb = int(next(args)) + if _gdb == 1: + pytest_prefix = 'gdb' + warnings += f'{arg=} is deprecated, use `-T gdb`.' + elif arg in ('-h', '--help'): - show_help() - return + show_help = True + elif arg == '-i': implementations = next(args) - elif arg in ('--mupdf', '-m'): - mupdf = next(args) - if not mupdf.startswith('git:') and '://' not in mupdf and mupdf != '-': - assert os.path.isdir(mupdf), f'Not a directory: {mupdf=}.' - mupdf = os.path.abspath(mupdf) - os.environ['PYMUPDF_SETUP_MUPDF_BUILD'] = mupdf + elif arg == '-k': - pytest_k = next(args) + pytest_options += f' -k {shlex.quote(next(args))}' + + elif arg in ('-m', '--mupdf'): + _mupdf = next(args) + if _mupdf == '-': + _mupdf = None + elif _mupdf.startswith('git:') or '://' in _mupdf: + os.environ['PYMUPDF_SETUP_MUPDF_BUILD'] = _mupdf + else: + assert os.path.isdir(_mupdf), f'Not a directory: {_mupdf=}' + os.environ['PYMUPDF_SETUP_MUPDF_BUILD'] = os.path.abspath(_mupdf) + mupdf_sync = _mupdf + + elif arg in ('-M', '--build-mupdf'): + env_extra['PYMUPDF_SETUP_MUPDF_REBUILD'] = next(args) + + elif arg == '-o': + os_names += next(args).split(',') + elif arg == '-p': - pytest_options = next(args) + pytest_options += f' {next(args)}' + elif arg == '-P': - packages = int(next(args)) - elif arg == '-s': - value = next(args) - assert value in ('0', '1'), f'`-s` must be followed by `0` or `1`, not {value=}.' - os.environ['PYMUPDF_SETUP_PY_LIMITED_API'] = value - elif arg == '--pytest-prefix': - pytest_prefix = next(args) + system_packages = int(next(args)) + elif arg == '--pybind': pybind = int(next(args)) + + elif arg == '--pyodide-build-version': + pyodide_build_version = next(args) + + elif arg == '-s': + _value = next(args) + assert _value in ('0', '1'), f'`-s` must be followed by `0` or `1`, not {_value=}.' + env_extra['PYMUPDF_SETUP_PY_LIMITED_API'] = _value + + elif arg == '--sync-paths': + sync_paths = True + elif arg == '--system-site-packages': system_site_packages = int(next(args)) + elif arg == '-t': test_names += next(args).split(',') + elif arg == '--timeout': - timeout = float(next(args)) + test_timeout = float(next(args)) + + elif arg in ('-T', '--pytest-prefix'): + pytest_prefix = next(args) + elif arg == '-v': venv = int(next(args)) - elif arg == '--build-flavour': - build_flavour = next(args) - elif arg == '--build-mupdf': - build_mupdf = int(next(args)) - elif arg == '--gdb': - gdb = int(next(args)) + assert venv in (0, 1, 2), f'Invalid {venv=} should be 0, 1 or 2.' + elif arg == '--valgrind': - valgrind = int(next(args)) - elif arg == '--valgrind-args': - valgrind_args = next(args) - elif arg == '--pyodide-build-version': - pyodide_build_version = next(args) - elif arg == '--pymupdf-pypi': - pymupdf_pypi = next(args) + _valgrind = int(next(args)) + if _valgrind == 1: + pytest_prefix = 'valgrind' + warnings += f'{arg=} is deprecated, use `-T _valgrind`.' + + elif arg in ('build', 'cibw', 'pyodide', 'test', 'wheel'): + commands.append(arg) + + elif arg == 'buildtest': + commands += ['build', 'test'] + + elif arg == 'install': + _pymupdf = next(args) + commands.append(f'{arg}.{_pymupdf}') + else: - assert 0, f'Unrecognised option: {arg=}.' + assert 0, f'Unrecognised option/command: {arg=}.' - if arg is None: - log(f'No command specified.') + # Handle special args --sync-paths, -h, -v, -o first. + # + if sync_paths: + # Just print required files, directories and checkouts. + print(pymupdf_dir) + if mupdf_sync: + print(mupdf_sync) + return + + if show_help: + print(__doc__) return - commands = list() - while 1: - assert arg in ('build', 'buildtest', 'test', 'wheel', 'pyodide_wheel'), \ - f'Unrecognised command: {arg=}.' - commands.append(arg) - try: - arg = next(args) - except StopIteration: - break - - venv_quick = (venv==1) + if os_names: + if platform.system().lower() not in os_names: + log(f'Not running because {platform.system().lower()=} not in {os_names=}') + return - # Run inside a venv. - if venv and sys.prefix == sys.base_prefix: - # We are not running in a venv. - log(f'Re-running in venv {gh_release.venv_name!r}.') - gh_release.venv( - ['python'] + argv, - quick=venv_quick, - system_site_packages=system_site_packages, - ) - return - - def do_build(wheel=False): - if pymupdf_pypi: - run(f'pip install --force-reinstall {pymupdf_pypi}') - else: - build( - build_type=build_type, - build_isolation=build_isolation, - venv_quick=venv_quick, - build_mupdf=build_mupdf, - build_flavour=build_flavour, - wheel=wheel, - ) - def do_test(): - test( - implementations=implementations, - valgrind=valgrind, - valgrind_args=valgrind_args, - venv_quick=venv_quick, - test_names=test_names, - pytest_options=pytest_options, - timeout=timeout, - gdb=gdb, - pytest_prefix=pytest_prefix, - test_fitz=test_fitz, - pytest_k=pytest_k, - pybind=pybind, - packages=packages, - ) + if commands: + if venv: + # Rerun ourselves inside a venv if not already in a venv. + if not venv_in(): + e = venv_run( + sys.argv, + f'venv-pymupdf-{platform.python_version()}-{int.bit_length(sys.maxsize+1)}', + recreate=(venv==2), + ) + sys.exit(e) + else: + log(f'Warning, no commands specified so nothing to do.') + # Handle commands. + # + have_installed = False for command in commands: + if 0: pass - elif command == 'build': - do_build() + + elif command in ('build', 'wheel'): + build( + env_extra, + build_isolation=build_isolation, + venv=venv, + wheel=(command=='wheel'), + ) + have_installed = True + + elif command == 'cibw': + # Build wheel(s) with cibuildwheel. + cibuildwheel(env_extra, cibw_name, cibw_pyodide) + + elif command.startswith('install.'): + name = command.lstrip('install.') + run(f'pip install --force-reinstall {name}') + have_installed = True + elif command == 'test': - do_test() - elif command == 'buildtest': - do_build() - do_test() - elif command == 'wheel': - do_build(wheel=True) - elif command == 'pyodide_wheel': + if not have_installed: + log(f'## Warning: have not built/installed PyMuPDF; testing whatever is already installed.') + test( + env_extra=env_extra, + implementations=implementations, + test_names=test_names, + pytest_options=pytest_options, + test_timeout=test_timeout, + pytest_prefix=pytest_prefix, + test_fitz=test_fitz, + pybind=pybind, + system_packages=system_packages, + venv=venv, + ) + + elif command == 'pyodide': build_pyodide_wheel(pyodide_build_version=pyodide_build_version) + else: - assert 0 + assert 0, f'{command=}' def get_env_bool(name, default=0): @@ -406,25 +570,12 @@ def venv_info(pytest_args=None): def build( - build_type=None, - build_isolation=None, - venv_quick=False, - build_mupdf=True, - build_flavour='pb', - wheel=False, + env_extra, + *, + build_isolation, + venv, + wheel, ): - ''' - Args: - build_type: - See top-level option `-b`. - build_isolation: - See top-level option `--build-isolation`. - venv_quick: - See top-level option `-v`. - build_mupdf: - See top-level option `build-mupdf` - ''' - print(f'{build_type=}') print(f'{build_isolation=}') if build_isolation is None: @@ -446,24 +597,92 @@ def build( del sys.path[0] if names: names = ' '.join(names) - if venv_quick: - log(f'{venv_quick=}: Not installing packages with pip: {names}') - else: + if venv == 2: run( f'python -m pip install --upgrade {names}') + else: + log(f'{venv=}: Not installing packages with pip: {names}') build_isolation_text = ' --no-build-isolation' - env_extra = dict() - if not build_mupdf: - env_extra['PYMUPDF_SETUP_MUPDF_REBUILD'] = '0' - if build_type: - env_extra['PYMUPDF_SETUP_MUPDF_BUILD_TYPE'] = build_type - if build_flavour: - env_extra['PYMUPDF_SETUP_FLAVOUR'] = build_flavour if wheel: - run(f'pip wheel{build_isolation_text} -v {pymupdf_dir}', env_extra=env_extra) + new_files = pipcl.NewFiles(f'wheelhouse/*.whl') + run(f'pip wheel{build_isolation_text} -w wheelhouse -v {pymupdf_dir_abs}', env_extra=env_extra) + wheel = new_files.get_one() + run(f'pip install --force-reinstall {wheel}') else: - run(f'pip install{build_isolation_text} -v {pymupdf_dir}', env_extra=env_extra) + run(f'pip install{build_isolation_text} -v --force-reinstall {pymupdf_dir_abs}', env_extra=env_extra) + + +def cibuildwheel(env_extra, cibw_name, cibw_pyodide): + run(f'pip install --upgrade {cibw_name}') + + # Some general flags. + env_extra['CIBW_BUILD_VERBOSITY'] = '1' + env_extra['CIBW_SKIP'] = 'pp* *i686 cp36* cp37* *musllinux* *-win32 *-aarch64' + + # Set what wheels to build, if not already specified. + if os.environ.get('CIBW_ARCHS') is None: + if os.environ.get('CIBW_ARCHS_WINDOWS') is None: + env_extra['CIBW_ARCHS_WINDOWS'] = 'auto64' + + if os.environ.get('CIBW_ARCHS_MACOS') is None: + env_extra['CIBW_ARCHS_MACOS'] = 'auto64' + + if os.environ.get('CIBW_ARCHS_LINUX') is None: + env_extra['CIBW_ARCHS_LINUX'] = 'auto64' + if os.environ.get('GITHUB_ACTIONS') == 'true': + # Special case to use emulation/cross-compilation of + # aarch64 on Linux. + env_extra['CIBW_ARCHS_LINUX'] += ' aarch64' + + # Tell cibuildwheel not to use `auditwheel` on Linux and MacOS, + # because it cannot cope with us deliberately having required + # libraries in different wheel - specifically in the PyMuPDF wheel. + # + # We cannot use a subset of auditwheel's functionality + # with `auditwheel addtag` because it says `No tags + # to be added` and terminates with non-zero. See: + # https://github.com/pypa/auditwheel/issues/439. + # + env_extra['CIBW_REPAIR_WHEEL_COMMAND_LINUX'] = '' + env_extra['CIBW_REPAIR_WHEEL_COMMAND_MACOS'] = '' + + # Tell cibuildwheel how to test PyMuPDF. + env_extra['CIBW_TEST_COMMAND'] = f'python {{project}}/scripts/test.py test' + + # Specify python versions. + CIBW_BUILD = env_extra.get('CIBW_BUILD') + log(f'{CIBW_BUILD=}') + if CIBW_BUILD is None: + if os.environ.get('GITHUB_ACTIONS') == 'true': + # Build/test all supported Python versions. + CIBW_BUILD = 'cp39* cp310* cp311* cp312* cp313*' + else: + # Build/test current Python only. + v = platform.python_version_tuple()[:2] + log(f'{v=}') + CIBW_BUILD = f'cp{"".join(v)}*' + # Pass all the environment variables we have set, to Linux + # docker. Note that this will miss any settings in the original + # environment. + env_extra['CIBW_ENVIRONMENT_PASS_LINUX'] = ' '.join(sorted(env_extra.keys())) + + # Build for lowest (assumed first) Python version. + # + cibw_pyodide_arg = ' --platform pyodide' if cibw_pyodide else '' + CIBW_BUILD_0 = CIBW_BUILD.split()[0] + log(f'Building for first Python version {CIBW_BUILD_0}.') + env_extra['CIBW_BUILD'] = CIBW_BUILD_0 + run(f'cd {pymupdf_dir} && cibuildwheel{cibw_pyodide_arg}', env_extra=env_extra) + + # Tell cibuildwheel to build and test all specified Python versions; it + # will notice that the wheel we built above supports all versions of + # Python, so will not actually do any builds here. + # + env_extra['CIBW_BUILD'] = CIBW_BUILD + run(f'cd {pymupdf_dir} && cibuildwheel{cibw_pyodide_arg}', env_extra=env_extra) + run(f'ls -ld {pymupdf_dir}/wheelhouse/*') + def build_pyodide_wheel(pyodide_build_version=None): ''' @@ -601,41 +820,19 @@ def pyodide_setup( def test( + *, + env_extra, implementations, - valgrind, - valgrind_args, - venv_quick=False, + venv=False, test_names=None, pytest_options=None, - timeout=None, - gdb=False, + test_timeout=None, pytest_prefix=None, test_fitz=True, pytest_k=None, pybind=False, - packages=False, + system_packages=False, ): - ''' - Args: - implementations: - See top-level option `-i`. - valgrind: - See top-level option `--valgrind`. - valgrind_args: - See top-level option `--valgrind-args`. - venv_quick: - . - test_names: - See top-level option `-t`. - pytest_options: - See top-level option `-p`. - gdb: - See top-level option `--gdb`. - pytest_prefix: - See top-level option `--pytest-prefix`. - test_fitz: - See top-level option `-f`. - ''' if pybind: cpp_path = 'pymupdf_test_pybind.cpp' cpp_exe = 'pymupdf_test_pybind.exe' @@ -711,73 +908,95 @@ def getmtime(path): pytest_arg += f' {pymupdf_dir_rel}' python = gh_release.relpath(sys.executable) log('Running tests with tests/run_compound.py and pytest.') - try: - if venv_quick: - log(f'{venv_quick=}: Not installing test packages: {gh_release.test_packages}') - else: - run(f'pip install --upgrade {gh_release.test_packages}') - run_compound_args = '' - if implementations: - run_compound_args += f' -i {implementations}' - if timeout: - run_compound_args += f' -t {timeout}' - env_extra = None - if valgrind: - if packages: - log('Installing valgrind.') - run(f'sudo apt update') - run(f'sudo apt install --upgrade valgrind') - run(f'valgrind --version') - - log('Running PyMuPDF tests under valgrind.') - command = ( - f'{python} {pymupdf_dir_rel}/tests/run_compound.py{run_compound_args}' - f' valgrind --suppressions={pymupdf_dir_rel}/valgrind.supp --error-exitcode=100 --errors-for-leak-kinds=none --fullpath-after= {valgrind_args}' - f' {python} -m pytest {pytest_options}{pytest_arg}' - ) - env_extra=dict( - PYTHONMALLOC='malloc', - PYMUPDF_RUNNING_ON_VALGRIND='1', + + if venv == 2: + run(f'pip install --upgrade {gh_release.test_packages}') + else: + log(f'{venv=}: Not installing test packages: {gh_release.test_packages}') + run_compound_args = '' + + if implementations: + run_compound_args += f' -i {implementations}' + + if test_timeout: + run_compound_args += f' -t {test_timeout}' + + if pytest_prefix in ('valgrind', 'helgrind'): + if system_packages: + log('Installing valgrind.') + run(f'sudo apt update') + run(f'sudo apt install --upgrade valgrind') + run(f'valgrind --version') + + command = f'{python} {pymupdf_dir_rel}/tests/run_compound.py{run_compound_args}' + + if pytest_prefix is None: + pass + elif pytest_prefix == 'gdb': + command += ' gdb --args' + elif pytest_prefix == 'valgrind': + env_extra['PYMUPDF_RUNNING_ON_VALGRIND'] = '1' + env_extra['PYTHONMALLOC'] = 'malloc' + command += ( + f' valgrind' + f' --suppressions={pymupdf_dir_abs}/valgrind.supp' + f' --trace-children=yes' + f' --num-callers=20' + f' --error-exitcode=100' + f' --errors-for-leak-kinds=none' + f' --fullpath-after=' ) - elif gdb: - command = f'{python} {pymupdf_dir_rel}/tests/run_compound.py{run_compound_args} gdb --args {python} -m pytest {pytest_options} {pytest_arg}' - elif pytest_prefix: - command = f'{python} {pymupdf_dir_rel}/tests/run_compound.py{run_compound_args} {pytest_prefix} {python} -m pytest {pytest_options} {pytest_arg}' - elif platform.system() == 'Windows': - # `python -m pytest` doesn't seem to work. - command = f'{python} {pymupdf_dir_rel}/tests/run_compound.py{run_compound_args} pytest {pytest_options} {pytest_arg}' - else: - # On OpenBSD `pip install pytest` doesn't seem to install the pytest - # command, so we use `python -m pytest ...`. - command = f'{python} {pymupdf_dir_rel}/tests/run_compound.py{run_compound_args} {python} -m pytest {pytest_options} {pytest_arg}' - - # Always start by removing any test_*_fitz.py files. - for p in glob.glob(f'{pymupdf_dir_rel}/tests/test_*_fitz.py'): - print(f'Removing {p=}') - os.remove(p) - if test_fitz: - # Create copies of each test file, modified to use `pymupdf` - # instead of `fitz`. - for p in glob.glob(f'{pymupdf_dir_rel}/tests/test_*.py'): - if os.path.basename(p).startswith('test_fitz_'): - # Don't recursively generate test_fitz_fitz_foo.py, - # test_fitz_fitz_fitz_foo.py, ... etc. - continue - branch, leaf = os.path.split(p) - p2 = f'{branch}/{leaf[:5]}fitz_{leaf[5:]}' - print(f'Converting {p=} to {p2=}.') - with open(p, encoding='utf8') as f: - text = f.read() - text2 = re.sub("([^\'])\\bpymupdf\\b", '\\1fitz', text) - if p.replace(os.sep, '/') == f'{pymupdf_dir_rel}/tests/test_docs_samples.py'.replace(os.sep, '/'): - assert text2 == text - else: - assert text2 != text, f'Unexpectedly unchanged when creating {p!r} => {p2!r}' - with open(p2, 'w', encoding='utf8') as f: - f.write(text2) - + elif pytest_prefix == 'helgrind': + env_extra['PYMUPDF_RUNNING_ON_VALGRIND'] = '1' + env_extra['PYTHONMALLOC'] = 'malloc' + command = ( + f' valgrind' + f' --tool=helgrind' + f' --trace-children=yes' + f' --num-callers=20' + f' --error-exitcode=100' + f' --fullpath-after=' + ) + else: + assert 0, f'Unrecognised {pytest_prefix=}' + + if platform.system() == 'Windows': + # `python -m pytest` doesn't seem to work. + command += ' pytest' + else: + # On OpenBSD `pip install pytest` doesn't seem to install the pytest + # command, so we use `python -m pytest ...`. + command += f' {python} -m pytest' + + command += f' {pytest_options} {pytest_arg}' + + # Always start by removing any test_*_fitz.py files. + for p in glob.glob(f'{pymupdf_dir_rel}/tests/test_*_fitz.py'): + print(f'Removing {p=}') + os.remove(p) + if test_fitz: + # Create copies of each test file, modified to use `pymupdf` + # instead of `fitz`. + for p in glob.glob(f'{pymupdf_dir_rel}/tests/test_*.py'): + if os.path.basename(p).startswith('test_fitz_'): + # Don't recursively generate test_fitz_fitz_foo.py, + # test_fitz_fitz_fitz_foo.py, ... etc. + continue + branch, leaf = os.path.split(p) + p2 = f'{branch}/{leaf[:5]}fitz_{leaf[5:]}' + print(f'Converting {p=} to {p2=}.') + with open(p, encoding='utf8') as f: + text = f.read() + text2 = re.sub("([^\'])\\bpymupdf\\b", '\\1fitz', text) + if p.replace(os.sep, '/') == f'{pymupdf_dir_rel}/tests/test_docs_samples.py'.replace(os.sep, '/'): + assert text2 == text + else: + assert text2 != text, f'Unexpectedly unchanged when creating {p!r} => {p2!r}' + with open(p2, 'w', encoding='utf8') as f: + f.write(text2) + try: log(f'Running tests with tests/run_compound.py and pytest.') - run(command, env_extra=env_extra, timeout=timeout) + run(command, env_extra=env_extra, timeout=test_timeout) except subprocess.TimeoutExpired as e: log(f'Timeout when running tests.') @@ -831,6 +1050,43 @@ def wrap_get_requires_for_build_wheel(dir_): return ' '.join(ret) +def venv_in(path=None): + ''' + If path is None, returns true if we are in a venv. Otherwise returns true + only if we are in venv . + ''' + if path: + return os.path.abspath(sys.prefix) == os.path.abspath(path) + else: + return sys.prefix != sys.base_prefix + + +def venv_run(args, path, recreate=True): + ''' + Runs command inside venv and returns termination code. + + Args: + args: + List of args. + path: + Name of venv. + recreate: + If false we do not run ` -m venv ` if + already exists. This avoids a delay in the common case where + is already set up, but fails if exists but does not contain + a valid venv. + ''' + if recreate or not os.path.isdir(path): + run(f'{sys.executable} -m venv {path}') + if platform.system() == 'Windows': + command = f'{path}\\Scripts\\activate' + else: + command = f'. {path}/bin/activate' + command += f' && python {shlex.join(args)}' + e = run(command, check=0) + return e + + if __name__ == '__main__': try: sys.exit(main(sys.argv)) diff --git a/setup.py b/setup.py index 01bc391fd..e8c958ecb 100755 --- a/setup.py +++ b/setup.py @@ -194,6 +194,7 @@ import subprocess import sys import tarfile +import traceback import urllib.request import zipfile @@ -221,21 +222,7 @@ def run(command, check=1): if 1: # For debugging. log(f'### Starting.') - log(f'__name__: {__name__!r}') - log(f'platform.platform(): {platform.platform()!r}') - log(f'platform.python_version(): {platform.python_version()!r}') - log(f'sys.executable: {sys.executable!r}') - log(f'CPU bits: {32 if sys.maxsize == 2**31 - 1 else 64} {sys.maxsize=}') - log(f'__file__: {__file__!r}') - log(f'os.getcwd(): {os.getcwd()!r}') - log(f'getconf ARG_MAX: {pipcl.run("getconf ARG_MAX", capture=1, check=0, verbose=0)!r}') - log(f'sys.argv ({len(sys.argv)}):') - for i, arg in enumerate(sys.argv): - log(f' {i}: {arg!r}') - log(f'os.environ ({len(os.environ)}):') - for k in sorted( os.environ.keys()): - v = os.environ[ k] - log( f' {k}: {v!r}') + pipcl.show_system() PYMUPDF_SETUP_FLAVOUR = os.environ.get( 'PYMUPDF_SETUP_FLAVOUR', 'pbd') @@ -802,7 +789,18 @@ def build_mupdf_windows( #log( f'Building mupdf.') devenv = os.environ.get('PYMUPDF_SETUP_DEVENV') if not devenv: - vs = pipcl.wdev.WindowsVS() + try: + # Prefer VS-2019 as that is what MuPDF's project/solution files are + # written for. + log(f'Looking for Visual Studio 2019.') + vs = pipcl.wdev.WindowsVS(year=2019) + except Exception as e: + log(f'Failed to find VS-2019:\n' + f'{textwrap.indent(traceback.format_exc(), " ")}' + ) + log(f'Looking for any Visual Studio.') + vs = pipcl.wdev.WindowsVS() + log(f'vs:\n{vs.description_ml(" ")}') devenv = vs.devenv if not devenv: devenv = 'devenv.com' @@ -1229,15 +1227,15 @@ def sdist(): # # PyMuPDF version. -version_p = '1.26.0' +version_p = '1.26.1' + +version_mupdf = '1.26.2' # PyMuPDFb version. This is the PyMuPDF version whose PyMuPDFb wheels we will # (re)use if generating separate PyMuPDFb wheels. Though as of PyMuPDF-1.24.11 # (2024-10-03) we no longer use PyMuPDFb wheels so this is actually unused. # -version_b = '1.26.0' - -version_mupdf = '1.26.1' +version_b = '1.26.1' if os.path.exists(f'{g_root}/{g_pymupdfb_sdist_marker}'): diff --git a/src/__init__.py b/src/__init__.py index 8d48ce717..f389fa043 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -11408,621 +11408,6 @@ def width(self): tr = top_right -class Shape: - """Create a new shape.""" - - def __init__(self, page: Page): - CheckParent(page) - self.page = page - self.doc = page.parent - if not self.doc.is_pdf: - raise ValueError("not a PDF") - self.height = page.mediabox_size.y - self.width = page.mediabox_size.x - self.x = page.cropbox_position.x - self.y = page.cropbox_position.y - - self.pctm = page.transformation_matrix # page transf. matrix - self.ipctm = ~self.pctm # inverted transf. matrix - - self.draw_cont = "" - self.text_cont = "" - self.totalcont = "" - self.last_point = None - self.rect = None - - def commit(self, overlay: bool = True) -> None: - """ - Update the page's /Contents object with Shape data. The argument - controls whether data appear in foreground (default) or background. - """ - CheckParent(self.page) # doc may have died meanwhile - self.totalcont += self.text_cont - - self.totalcont = self.totalcont.encode() - - if self.totalcont != b"": - # make /Contents object with dummy stream - xref = TOOLS._insert_contents(self.page, b" ", overlay) - # update it with potential compression - mupdf.pdf_update_stream(self.doc, xref, self.totalcont) - - self.last_point = None # clean up ... - self.rect = None # - self.draw_cont = "" # for potential ... - self.text_cont = "" # ... - self.totalcont = "" # re-use - return - - def draw_bezier( - self, - p1: point_like, - p2: point_like, - p3: point_like, - p4: point_like, - ):# -> Point: - """Draw a standard cubic Bezier curve.""" - p1 = Point(p1) - p2 = Point(p2) - p3 = Point(p3) - p4 = Point(p4) - if not (self.last_point == p1): - args = JM_TUPLE(p1 * self.ipctm) - self.draw_cont += f"{_format_g(args)} m\n" - args = JM_TUPLE(list(p2 * self.ipctm) + list(p3 * self.ipctm) + list(p4 * self.ipctm)) - self.draw_cont += f"{_format_g(args)} c\n" - self.updateRect(p1) - self.updateRect(p2) - self.updateRect(p3) - self.updateRect(p4) - self.last_point = p4 - return self.last_point - - def draw_circle(self, center: point_like, radius: float):# -> Point: - """Draw a circle given its center and radius.""" - if not radius > EPSILON: - raise ValueError("radius must be positive") - center = Point(center) - p1 = center - (radius, 0) - return self.draw_sector(center, p1, 360, fullSector=False) - - def draw_curve( - self, - p1: point_like, - p2: point_like, - p3: point_like, - ):# -> Point: - """Draw a curve between points using one control point.""" - kappa = 0.55228474983 - p1 = Point(p1) - p2 = Point(p2) - p3 = Point(p3) - k1 = p1 + (p2 - p1) * kappa - k2 = p3 + (p2 - p3) * kappa - return self.draw_bezier(p1, k1, k2, p3) - - def draw_line(self, p1: point_like, p2: point_like):# -> Point: - """Draw a line between two points.""" - p1 = Point(p1) - p2 = Point(p2) - if not (self.last_point == p1): - self.draw_cont += _format_g(JM_TUPLE(p1 * self.ipctm)) + " m\n" - self.last_point = p1 - self.updateRect(p1) - - self.draw_cont += _format_g(JM_TUPLE(p2 * self.ipctm)) + " l\n" - self.updateRect(p2) - self.last_point = p2 - return self.last_point - - def draw_oval(self, tetra: typing.Union[quad_like, rect_like]):# -> Point: - """Draw an ellipse inside a tetrapod.""" - if len(tetra) != 4: - raise ValueError("invalid arg length") - if hasattr(tetra[0], "__float__"): - q = Rect(tetra).quad - else: - q = Quad(tetra) - - mt = q.ul + (q.ur - q.ul) * 0.5 - mr = q.ur + (q.lr - q.ur) * 0.5 - mb = q.ll + (q.lr - q.ll) * 0.5 - ml = q.ul + (q.ll - q.ul) * 0.5 - if not (self.last_point == ml): - self.draw_cont += _format_g(JM_TUPLE(ml * self.ipctm)) + " m\n" - self.last_point = ml - self.draw_curve(ml, q.ll, mb) - self.draw_curve(mb, q.lr, mr) - self.draw_curve(mr, q.ur, mt) - self.draw_curve(mt, q.ul, ml) - self.updateRect(q.rect) - self.last_point = ml - return self.last_point - - def draw_polyline(self, points: list):# -> Point: - """Draw several connected line segments.""" - for i, p in enumerate(points): - if i == 0: - if not (self.last_point == Point(p)): - self.draw_cont += _format_g(JM_TUPLE(Point(p) * self.ipctm)) + " m\n" - self.last_point = Point(p) - else: - self.draw_cont += _format_g(JM_TUPLE(Point(p) * self.ipctm)) + " l\n" - self.updateRect(p) - - self.last_point = Point(points[-1]) - return self.last_point - - def draw_quad(self, quad: quad_like):# -> Point: - """Draw a Quad.""" - q = Quad(quad) - return self.draw_polyline([q.ul, q.ll, q.lr, q.ur, q.ul]) - - def draw_rect(self, rect: rect_like):# -> Point: - """Draw a rectangle.""" - r = Rect(rect) - args = JM_TUPLE(list(r.bl * self.ipctm) + [r.width, r.height]) - self.draw_cont += _format_g(args) + " re\n" - - self.updateRect(r) - self.last_point = r.tl - return self.last_point - - def draw_sector( - self, - center: point_like, - point: point_like, - beta: float, - fullSector: bool = True, - ):# -> Point: - """Draw a circle sector.""" - center = Point(center) - point = Point(point) - def l3(a, b): - return _format_g((a, b)) + " m\n" - def l4(a, b, c, d, e, f): - return _format_g((a, b, c, d, e, f)) + " c\n" - def l5(a, b): - return _format_g((a, b)) + " l\n" - betar = math.radians(-beta) - w360 = math.radians(math.copysign(360, betar)) * (-1) - w90 = math.radians(math.copysign(90, betar)) - w45 = w90 / 2 - while abs(betar) > 2 * math.pi: - betar += w360 # bring angle below 360 degrees - if not (self.last_point == point): - self.draw_cont += l3(JM_TUPLE(point * self.ipctm)) - self.last_point = point - Q = Point(0, 0) # just make sure it exists - C = center - P = point - S = P - C # vector 'center' -> 'point' - rad = abs(S) # circle radius - - if not rad > EPSILON: - raise ValueError("radius must be positive") - - alfa = self.horizontal_angle(center, point) - while abs(betar) > abs(w90): # draw 90 degree arcs - q1 = C.x + math.cos(alfa + w90) * rad - q2 = C.y + math.sin(alfa + w90) * rad - Q = Point(q1, q2) # the arc's end point - r1 = C.x + math.cos(alfa + w45) * rad / math.cos(w45) - r2 = C.y + math.sin(alfa + w45) * rad / math.cos(w45) - R = Point(r1, r2) # crossing point of tangents - kappah = (1 - math.cos(w45)) * 4 / 3 / abs(R - Q) - kappa = kappah * abs(P - Q) - cp1 = P + (R - P) * kappa # control point 1 - cp2 = Q + (R - Q) * kappa # control point 2 - self.draw_cont += l4(JM_TUPLE( - list(cp1 * self.ipctm) + list(cp2 * self.ipctm) + list(Q * self.ipctm) - )) - - betar -= w90 # reduce param angle by 90 deg - alfa += w90 # advance start angle by 90 deg - P = Q # advance to arc end point - # draw (remaining) arc - if abs(betar) > 1e-3: # significant degrees left? - beta2 = betar / 2 - q1 = C.x + math.cos(alfa + betar) * rad - q2 = C.y + math.sin(alfa + betar) * rad - Q = Point(q1, q2) # the arc's end point - r1 = C.x + math.cos(alfa + beta2) * rad / math.cos(beta2) - r2 = C.y + math.sin(alfa + beta2) * rad / math.cos(beta2) - R = Point(r1, r2) # crossing point of tangents - # kappa height is 4/3 of segment height - kappah = (1 - math.cos(beta2)) * 4 / 3 / abs(R - Q) # kappa height - kappa = kappah * abs(P - Q) / (1 - math.cos(betar)) - cp1 = P + (R - P) * kappa # control point 1 - cp2 = Q + (R - Q) * kappa # control point 2 - self.draw_cont += l4(JM_TUPLE( - list(cp1 * self.ipctm) + list(cp2 * self.ipctm) + list(Q * self.ipctm) - )) - if fullSector: - self.draw_cont += l3(JM_TUPLE(point * self.ipctm)) - self.draw_cont += l5(JM_TUPLE(center * self.ipctm)) - self.draw_cont += l5(JM_TUPLE(Q * self.ipctm)) - self.last_point = Q - return self.last_point - - def draw_squiggle( - self, - p1: point_like, - p2: point_like, - breadth=2, - ):# -> Point: - """Draw a squiggly line from p1 to p2.""" - p1 = Point(p1) - p2 = Point(p2) - S = p2 - p1 # vector start - end - rad = abs(S) # distance of points - cnt = 4 * int(round(rad / (4 * breadth), 0)) # always take full phases - if cnt < 4: - raise ValueError("points too close") - mb = rad / cnt # revised breadth - matrix = Matrix(TOOLS._hor_matrix(p1, p2)) # normalize line to x-axis - i_mat = ~matrix # get original position - k = 2.4142135623765633 # y of draw_curve helper point - - points = [] # stores edges - for i in range(1, cnt): - if i % 4 == 1: # point "above" connection - p = Point(i, -k) * mb - elif i % 4 == 3: # point "below" connection - p = Point(i, k) * mb - else: # else on connection line - p = Point(i, 0) * mb - points.append(p * i_mat) - - points = [p1] + points + [p2] - cnt = len(points) - i = 0 - while i + 2 < cnt: - self.draw_curve(points[i], points[i + 1], points[i + 2]) - i += 2 - return p2 - - def draw_zigzag( - self, - p1: point_like, - p2: point_like, - breadth: float = 2, - ):# -> Point: - """Draw a zig-zagged line from p1 to p2.""" - p1 = Point(p1) - p2 = Point(p2) - S = p2 - p1 # vector start - end - rad = abs(S) # distance of points - cnt = 4 * int(round(rad / (4 * breadth), 0)) # always take full phases - if cnt < 4: - raise ValueError("points too close") - mb = rad / cnt # revised breadth - matrix = Matrix(TOOLS._hor_matrix(p1, p2)) # normalize line to x-axis - i_mat = ~matrix # get original position - points = [] # stores edges - for i in range(1, cnt): - if i % 4 == 1: # point "above" connection - p = Point(i, -1) * mb - elif i % 4 == 3: # point "below" connection - p = Point(i, 1) * mb - else: # ignore others - continue - points.append(p * i_mat) - self.draw_polyline([p1] + points + [p2]) # add start and end points - return p2 - - def finish( - self, - width: float = 1, - color: OptSeq = (0,), - fill: OptSeq = None, - lineCap: int = 0, - lineJoin: int = 0, - dashes: OptStr = None, - even_odd: bool = False, - morph: OptSeq = None, - closePath: bool = True, - fill_opacity: float = 1, - stroke_opacity: float = 1, - oc: int = 0, - ) -> None: - """Finish the current drawing segment. - - Notes: - Apply colors, opacity, dashes, line style and width, or - morphing. Also whether to close the path - by connecting last to first point. - """ - if self.draw_cont == "": # treat empty contents as no-op - return - - if width == 0: # border color makes no sense then - color = None - elif color is None: # vice versa - width = 0 - # if color == None and fill == None: - # raise ValueError("at least one of 'color' or 'fill' must be given") - color_str = ColorCode(color, "c") # ensure proper color string - fill_str = ColorCode(fill, "f") # ensure proper fill string - - optcont = self.page._get_optional_content(oc) - if optcont is not None: - self.draw_cont = "/OC /%s BDC\n" % optcont + self.draw_cont - emc = "EMC\n" - else: - emc = "" - - alpha = self.page._set_opacity(CA=stroke_opacity, ca=fill_opacity) - if alpha is not None: - self.draw_cont = "/%s gs\n" % alpha + self.draw_cont - - if width != 1 and width != 0: - self.draw_cont += _format_g(width) + " w\n" - - if lineCap != 0: - self.draw_cont = "%i J\n" % lineCap + self.draw_cont - if lineJoin != 0: - self.draw_cont = "%i j\n" % lineJoin + self.draw_cont - - if dashes not in (None, "", "[] 0"): - self.draw_cont = "%s d\n" % dashes + self.draw_cont - - if closePath: - self.draw_cont += "h\n" - self.last_point = None - - if color is not None: - self.draw_cont += color_str - - if fill is not None: - self.draw_cont += fill_str - if color is not None: - if not even_odd: - self.draw_cont += "B\n" - else: - self.draw_cont += "B*\n" - else: - if not even_odd: - self.draw_cont += "f\n" - else: - self.draw_cont += "f*\n" - else: - self.draw_cont += "S\n" - - self.draw_cont += emc - if CheckMorph(morph): - m1 = Matrix( - 1, 0, 0, 1, morph[0].x + self.x, self.height - morph[0].y - self.y - ) - mat = ~m1 * morph[1] * m1 - self.draw_cont = _format_g(JM_TUPLE(mat) + self.draw_cont) + " cm\n" - - self.totalcont += "\nq\n" + self.draw_cont + "Q\n" - self.draw_cont = "" - self.last_point = None - return - - @staticmethod - def horizontal_angle(C, P): - """Return the angle to the horizontal for the connection from C to P. - This uses the arcus sine function and resolves its inherent ambiguity by - looking up in which quadrant vector S = P - C is located. - """ - S = Point(P - C).unit # unit vector 'C' -> 'P' - alfa = math.asin(abs(S.y)) # absolute angle from horizontal - if S.x < 0: # make arcsin result unique - if S.y <= 0: # bottom-left - alfa = -(math.pi - alfa) - else: # top-left - alfa = math.pi - alfa - else: - if S.y >= 0: # top-right - pass - else: # bottom-right - alfa = -alfa - return alfa - - def insert_text( - self, - point: point_like, - buffer_: typing.Union[str, list], - fontsize: float = 11, - lineheight: OptFloat = None, - fontname: str = "helv", - fontfile: OptStr = None, - set_simple: bool = 0, - encoding: int = 0, - color: OptSeq = None, - fill: OptSeq = None, - render_mode: int = 0, - border_width: float = 1, - rotate: int = 0, - morph: OptSeq = None, - stroke_opacity: float = 1, - fill_opacity: float = 1, - oc: int = 0, - ) -> int: - # ensure 'text' is a list of strings, worth dealing with - if not bool(buffer_): - return 0 - - if type(buffer_) not in (list, tuple): - text = buffer_.splitlines() - else: - text = buffer_ - - if not len(text) > 0: - return 0 - - point = Point(point) - try: - maxcode = max([ord(c) for c in " ".join(text)]) - except Exception: - exception_info() - return 0 - - # ensure valid 'fontname' - fname = fontname - if fname.startswith("/"): - fname = fname[1:] - - xref = self.page.insert_font( - fontname=fname, - fontfile=fontfile, - encoding=encoding, - set_simple=set_simple, - ) - fontinfo = CheckFontInfo(self.doc, xref) - - fontdict = fontinfo[1] - ordering = fontdict["ordering"] - simple = fontdict["simple"] - bfname = fontdict["name"] - ascender = fontdict["ascender"] - descender = fontdict["descender"] - if lineheight: - lheight = fontsize * lineheight - elif ascender - descender <= 1: - lheight = fontsize * 1.2 - else: - lheight = fontsize * (ascender - descender) - - if maxcode > 255: - glyphs = self.doc.get_char_widths(xref, maxcode + 1) - else: - glyphs = fontdict["glyphs"] - - tab = [] - for t in text: - if simple and bfname not in ("Symbol", "ZapfDingbats"): - g = None - else: - g = glyphs - tab.append(getTJstr(t, g, simple, ordering)) - text = tab - - color_str = ColorCode(color, "c") - fill_str = ColorCode(fill, "f") - if not fill and render_mode == 0: # ensure fill color when 0 Tr - fill = color - fill_str = ColorCode(color, "f") - - morphing = CheckMorph(morph) - rot = rotate - if rot % 90 != 0: - raise ValueError("bad rotate value") - - while rot < 0: - rot += 360 - rot = rot % 360 # text rotate = 0, 90, 270, 180 - - templ1 = lambda a, b, c, d, e, f, g: f"\nq\n{a}{b}BT\n%{c}1 0 0 1 {_format_g((d, e))} Tm\n/{f} {g} Tf " - templ2 = lambda a: f"TJ\n0 -{_format_g(a)} TD\n" - cmp90 = "0 1 -1 0 0 0 cm\n" # rotates 90 deg counter-clockwise - cmm90 = "0 -1 1 0 0 0 cm\n" # rotates 90 deg clockwise - cm180 = "-1 0 0 -1 0 0 cm\n" # rotates by 180 deg. - height = self.height - width = self.width - - # setting up for standard rotation directions - # case rotate = 0 - if morphing: - m1 = Matrix(1, 0, 0, 1, morph[0].x + self.x, height - morph[0].y - self.y) - mat = ~m1 * morph[1] * m1 - cm = _format_g(JM_TUPLE(mat)) + " cm\n" - else: - cm = "" - top = height - point.y - self.y # start of 1st char - left = point.x + self.x # start of 1. char - space = top # space available - if rot == 90: - left = height - point.y - self.y - top = -point.x - self.x - cm += cmp90 - space = width - abs(top) - - elif rot == 270: - left = -height + point.y + self.y - top = point.x + self.x - cm += cmm90 - space = abs(top) - - elif rot == 180: - left = -point.x - self.x - top = -height + point.y + self.y - cm += cm180 - space = abs(point.y + self.y) - - optcont = self.page._get_optional_content(oc) - if optcont is not None: - bdc = "/OC /%s BDC\n" % optcont - emc = "EMC\n" - else: - bdc = emc = "" - - alpha = self.page._set_opacity(CA=stroke_opacity, ca=fill_opacity) - if alpha is None: - alpha = "" - else: - alpha = "/%s gs\n" % alpha - nres = templ1(bdc, alpha, cm, left, top, fname, fontsize) - if render_mode > 0: - nres += "%i Tr " % render_mode - if border_width != 1: - nres += _format_g(border_width) + " w " - if color is not None: - nres += color_str - if fill is not None: - nres += fill_str - - # ========================================================================= - # start text insertion - # ========================================================================= - nres += text[0] - nlines = 1 # set output line counter - if len(text) > 1: - nres += templ2(lheight) # line 1 - else: - nres += 'TJ' - for i in range(1, len(text)): - if space < lheight: - break # no space left on page - if i > 1: - nres += "\nT* " - nres += text[i] + 'TJ' - space -= lheight - nlines += 1 - - nres += "\nET\n%sQ\n" % emc - - # ========================================================================= - # end of text insertion - # ========================================================================= - # update the /Contents object - self.text_cont += nres - return nlines - - def update_rect(self, x): - if self.rect is None: - if len(x) == 2: - self.rect = Rect(x, x) - else: - self.rect = Rect(x) - else: - if len(x) == 2: - x = Point(x) - self.rect.x0 = min(self.rect.x0, x.x) - self.rect.y0 = min(self.rect.y0, x.y) - self.rect.x1 = max(self.rect.x1, x.x) - self.rect.y1 = max(self.rect.y1, x.y) - else: - x = Rect(x) - self.rect.x0 = min(self.rect.x0, x.x0) - self.rect.y0 = min(self.rect.y0, x.y0) - self.rect.x1 = max(self.rect.x1, x.x1) - self.rect.y1 = max(self.rect.y1, x.y1) - - class Story: def __init__( self, html='', user_css=None, em=12, archive=None): diff --git a/tests/test_memory.py b/tests/test_memory.py index 7d37833db..48a3a1bd2 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -178,9 +178,9 @@ def get_stat(): drss = rss - state.prev state.prev = rss print(f'test_4125():' - f' rss={pipcl.number_sep(rss)}' - f' rss-rss0={pipcl.number_sep(rss-state.rsss[0])}' - f' drss={pipcl.number_sep(drss)}' + f' {rss=:,}' + f' rss-rss0={rss-state.rsss[0]:,}' + f' drss={drss:,}' f'.' ) diff --git a/wdev.py b/wdev.py index d2d1c86ac..33c348459 100644 --- a/wdev.py +++ b/wdev.py @@ -13,6 +13,7 @@ import pipcl + class WindowsVS: r''' Windows only. Finds locations of Visual Studio command-line tools. Assumes @@ -33,7 +34,16 @@ class WindowsVS: `.csc` is C# compiler; will be None if not found. ''' - def __init__( self, year=None, grade=None, version=None, cpu=None, verbose=False): + def __init__( + self, + *, + year=None, + grade=None, + version=None, + cpu=None, + directory=None, + verbose=False, + ): ''' Args: year: @@ -52,7 +62,15 @@ def __init__( self, year=None, grade=None, version=None, cpu=None, verbose=False variable WDEV_VS_VERSION if set. cpu: None or a `WindowsCpu` instance. + directory: + Ignore year, grade, version and cpu and use this directory + directly. + verbose: + . + ''' + if year is not None: + year = str(year) # Allow specification as a number. def default(value, name): if value is None: name2 = f'WDEV_VS_{name.upper()}' @@ -68,16 +86,17 @@ def default(value, name): if not cpu: cpu = WindowsCpu() - # Find `directory`. - # - pattern = f'C:\\Program Files*\\Microsoft Visual Studio\\{year if year else "2*"}\\{grade if grade else "*"}' - directories = glob.glob( pattern) - if verbose: - _log( f'Matches for: {pattern=}') - _log( f'{directories=}') - assert directories, f'No match found for: {pattern}' - directories.sort() - directory = directories[-1] + if not directory: + # Find `directory`. + # + pattern = _vs_pattern(year, grade) + directories = glob.glob( pattern) + if verbose: + _log( f'Matches for: {pattern=}') + _log( f'{directories=}') + assert directories, f'No match found for {pattern=}.' + directories.sort() + directory = directories[-1] # Find `devenv`. # @@ -167,7 +186,7 @@ def default(value, name): self.year = year self.cpu = cpu except Exception as e: - raise Exception( f'Unable to find Visual Studio') from e + raise Exception( f'Unable to find Visual Studio {year=} {grade=} {version=} {cpu=} {directory=}') from e def description_ml( self, indent=''): ''' @@ -189,7 +208,40 @@ def description_ml( self, indent=''): return textwrap.indent( ret, indent) def __repr__( self): - return ' '.join( self._description()) + items = list() + for name in ( + 'year', + 'grade', + 'version', + 'directory', + 'vcvars', + 'cl', + 'link', + 'csc', + 'msbuild', + 'devenv', + 'cpu', + ): + items.append(f'{name}={getattr(self, name)!r}') + return ' '.join(items) + + +def _vs_pattern(year=None, grade=None): + return f'C:\\Program Files*\\Microsoft Visual Studio\\{year if year else "2*"}\\{grade if grade else "*"}' + + +def windows_vs_multiple(year=None, grade=None, verbose=0): + ''' + Returns list of WindowsVS instances. + ''' + ret = list() + directories = glob.glob(_vs_pattern(year, grade)) + for directory in directories: + vs = WindowsVS(directory=directory) + if verbose: + _log(vs.description_ml()) + ret.append(vs) + return ret class WindowsCpu: