diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 55cc1b1b..ea89afaf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -182,7 +182,7 @@ jobs: CIBW_MUSLLINUX_X86_64_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_2' }} CIBW_MUSLLINUX_I686_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_2' }} CIBW_MUSLLINUX_AARCH64_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_2' }} - CIBW_TEST_REQUIRES: pytest setuptools # 3.12+ no longer includes distutils, just always ensure setuptools is present + CIBW_TEST_REQUIRES: pytest setuptools meson-python ninja # 3.12+ no longer includes distutils, just always ensure setuptools is present CIBW_TEST_COMMAND: PYTHONUNBUFFERED=1 python -m pytest ${{ matrix.test_args || '{project}' }} # default to test all run: | set -eux @@ -268,7 +268,7 @@ jobs: id: build env: CIBW_BUILD: ${{ matrix.spec }} - CIBW_TEST_REQUIRES: pytest setuptools + CIBW_TEST_REQUIRES: pytest setuptools meson-python ninja CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; PYTHONUNBUFFERED=1 pytest MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.13' }} SDKROOT: ${{ matrix.sdkroot || 'macosx' }} @@ -351,7 +351,7 @@ jobs: id: build env: CIBW_BUILD: ${{ matrix.spec }} - CIBW_TEST_REQUIRES: pytest setuptools + CIBW_TEST_REQUIRES: pytest setuptools meson-python ninja CIBW_TEST_COMMAND: ${{ matrix.test_cmd || 'python -m pytest {package}/src/c' }} # FIXME: /testing takes ~45min on Windows and has some failures... # CIBW_TEST_COMMAND='python -m pytest {package}/src/c {package}/testing' @@ -527,7 +527,7 @@ jobs: - name: build and install run: | - python -m pip install pytest setuptools pytest-run-parallel + python -m pip install pytest setuptools meson-python ninja pytest-run-parallel python -m pip install . - name: run tests under pytest-run-parallel @@ -542,13 +542,7 @@ jobs: - name: build and install run: | - # these two steps can be removed when - # https://github.com/nascheme/cpython_sanity/issues/12 and the images - # are rebuilt - apt update - apt install -y llvm - - python -m pip install setuptools pytest pytest-run-parallel + python -m pip install setuptools meson-python ninja pytest pytest-run-parallel CFLAGS="-g -O3 -fsanitize=thread" python -m pip install -v . - name: run tests under pytest-run-parallel diff --git a/MANIFEST.in b/MANIFEST.in index a7616ffe..0590e1e5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ recursive-include src/cffi *.py *.h recursive-include src/c *.c *.h *.asm *.py win64.obj ffi.lib -recursive-include testing *.py *.c *.h +recursive-include testing *.py *.c *.h meson.build pyproject.toml *.txt recursive-include doc *.py *.rst Makefile *.bat recursive-include demo py.cleanup *.py embedding_test.c manual.c include AUTHORS LICENSE setup.py setup_base.py diff --git a/doc/source/buildtool.rst b/doc/source/buildtool.rst new file mode 100644 index 00000000..1f696b27 --- /dev/null +++ b/doc/source/buildtool.rst @@ -0,0 +1,341 @@ +.. _buildtool_docs: + +========================================= +Building and Distributing CFFI Extensions +========================================= + +.. contents:: + +CFFI ships a command-line tool, ``gen-cffi-src``, that produces the +same output as :meth:`FFI.emit_c_code`: a ``.c`` source file ready to +be compiled into a CPython extension module. This tool enables +integrating with any build backend, such as `meson-python +`_, `scikit-build-core +`_, or similar. + +Installing CFFI installs the ``gen-cffi-src`` script; running ``python +-m cffi.buildtool`` invokes the same command line and behaves +identically. Use the former where a script with a Python shebang +makes more sense (e.g. cross-compiling) and use the latter when the +console script is not on PATH but a Python interpreter is. + +The rest of this page uses meson-python in the examples, but any `Python build +backend`_ that lets you run a helper program during the build can +drive ``gen-cffi-src`` the same way. + +The only way to use the buildtool functionality is via this command +line; the implementation inside the ``cffi`` package is private. + +The implementation is based on the `cffi-buildtool`_ project by Rose Davidson +(`@inklesspen`_ on GitHub). It is included in CFFI with permission of the original +author. + +.. _Python build backend: https://packaging.python.org/en/latest/guides/tool-recommendations/#build-backends-for-extension-modules +.. _cffi-buildtool: https://github.com/inklesspen/cffi-buildtool +.. _@inklesspen: https://github.com/inklesspen + +The ``gen-cffi-src`` Command-line Tool +====================================== + +``gen-cffi-src`` has two subcommands. The first, ``exec-python`` is +most useful if you already have a Python script that sets up an FFI +definition. The second, ``read-sources`` is most useful if you are wrapping +a large API surface and want a more structured way to specify a set of FFI +definitions. + +``gen-cffi-src exec-python`` +---------------------------- + +This mode takes a Python script that dynamically defines an FFI interface and +accompanying C extension source code. The FFI definition script is the same +script you would normally run by hand -- the one the CFFI docs show under +:ref:`real-example`. + +Let's say we want to create an extension module that wraps a single C function named +``square``. The ``square`` function has the following signature: + +.. code-block:: C + + int square(int n); + +Let's also say this function definition is exposed inside a header named +`square.h`. We could create a set of FFI bindings for this function given this +``_squared_build.py``:: + + from cffi import FFI + + ffibuilder = FFI() + + ffibuilder.cdef("int square(int n);") + + ffibuilder.set_source( + "squared._squared", + '#include "square.h"', + ) + +To generate the source code for the C extension, you would run: + +.. code-block:: console + + $ gen-cffi-src exec-python _squared_build.py _squared.c + +Many CFFI FFI definition scripts have an ``if __name__ == "__main__"`` section +that triggers a compilation step. This is not needed for a script run +by ``gen-cffi-src``, which does not generate compiled artifacts, +only C source code. It is up to your build-backend of choice +(e.g. meson-python) to run a C compiler and build compiled artifacts. +If the script does have such a section it is harmless: the script is +executed with ``__name__`` set to ``"cffi.buildtool"``, so the block is +skipped and an existing FFI definition script works unchanged. + +If the :class:`cffi.FFI` is bound to a name other than ``ffibuilder``, pass +``--ffi-var``. To make that concrete, let's say your FFI definition script +creates an FFI object named ``make_ffi``:: + + from cffi import FFI + + make_ffi = FFI() + +In that case, you would pass ``--ffi-var=make_ffi`` to ``gen-cffi-src``: + +.. code-block:: console + + $ gen-cffi-src exec-python --ffi-var=make_ffi _squared_build.py _squared.c + +.. note:: + + CFFI's setuptools integration supports passing ``libraries=``, + ``library_dirs=``, ``include_dirs=``, and ``extra_compile_args=`` + arguments to :meth:`FFI.set_source`. When using ``gen-cffi-src``, + these arguments are *ignored*. Link and include settings are the + build backend's responsibility; for meson-python you would express + them through the ``dependencies``, ``include_directories``, and + ``c_args`` arguments of ``py.extension_module()``. + +``gen-cffi-src read-sources`` +----------------------------- + +For larger modules, keeping the FFI definition and any necessary C +source prelude in separate files tends to be easier to work with -- +you can configure your editor to treat them as plain C, and write +presubmit tooling that parses the FFI definition directly without +extracting it from a Python script. + +Given ``squared.cdef.txt``: + +.. code-block:: C + + int square(int n); + +and ``squared.csrc.c``: + +.. code-block:: C + + #include "square.h" + +you would run the following command to generate a CFFI extension: + +.. code-block:: console + + $ gen-cffi-src read-sources squared._squared squared.cdef.txt squared.csrc.c _squared.c + +With all other details left exactly the same as the ``exec-python`` example. + +The first positional argument passed to the ``read-sources`` command is the +fully qualified module name that will be embedded in the generated C source +code (equivalent to the first argument to :meth:`FFI.set_source`). + + +A Worked Example Using ``meson-python`` +======================================= + +Project layout: + +.. code-block:: text + + squared/ + ├── pyproject.toml + ├── meson.build + └── src/ + ├── squared/ + │ ├── __init__.py + │ └── _squared_build.py + └── csrc/ + ├── square.h + └── square.c + +``pyproject.toml``: + +.. literalinclude:: ../../testing/cffi1/buildtool_examples/build_script_example/pyproject.toml + :language: toml + +``meson.build``: + +.. literalinclude:: ../../testing/cffi1/buildtool_examples/build_script_example/meson.build + :language: meson + +``src/squared/__init__.py``: + +.. literalinclude:: ../../testing/cffi1/buildtool_examples/build_script_example/src/squared/__init__.py + :language: python + +``src/squared/_squared_build.py``: + +.. literalinclude:: ../../testing/cffi1/buildtool_examples/build_script_example/src/squared/_squared_build.py + :language: python + +``src/csrc/square.h``: + +.. literalinclude:: ../../testing/cffi1/buildtool_examples/build_script_example/src/csrc/square.h + :language: C + +``src/csrc/square.c``: + +.. literalinclude:: ../../testing/cffi1/buildtool_examples/build_script_example/src/csrc/square.c + :language: C + +Build and install the project with any Python build front-end. For +example, with `pip`, in the root `squared` directory: + +.. code-block:: console + + $ python -m pip install . + $ python -c "from squared import squared; print(squared(7))" + 49 + +To switch this project to ``read-sources`` mode, replace +``_squared_build.py`` with two files, so that the project layout +becomes: + +.. code-block:: text + + squared/ + ├── pyproject.toml + ├── meson.build + └── src/ + ├── squared/ + │ ├── __init__.py + │ ├── squared.cdef.txt + │ └── squared.csrc.c + └── csrc/ + ├── square.h + └── square.c + +The first new file, ``squared.cdef.txt``, contains the FFI definition: + +.. literalinclude:: ../../testing/cffi1/buildtool_examples/cdef_example/src/squared/squared.cdef.txt + :language: python + +and the second, ``squared.csrc.c``, contains the C source prelude: + +.. literalinclude:: ../../testing/cffi1/buildtool_examples/cdef_example/src/squared/squared.csrc.c + :language: python + +then change two spots in the ``meson.build`` file. First, update the ``custom_target`` +``command`` to call ``gen-cffi-src read-sources`` with two input arguments: + +.. code-block:: meson + + command: [ + gen_cffi_src, + 'read-sources', + 'squared._squared', + '@INPUT0@', + '@INPUT1@', + '@OUTPUT@', + ], + +and then list both of the FFI specification files under ``input``: + +.. code-block:: meson + + input: ['src/squared/squared.cdef.txt', 'src/squared/squared.csrc.c'] + +Distributing CFFI Extensions using Setuptools +============================================= + +.. _distutils-setuptools: + + You can (but don't have to) use CFFI's **Distutils** or + **Setuptools integration** when writing a ``setup.py``. For + Distutils (only in out-of-line API mode; deprecated since + Python 3.10): + + .. code-block:: python + + # setup.py (requires CFFI to be installed first) + from distutils.core import setup + + import foo_build # possibly with sys.path tricks to find it + + setup( + ..., + ext_modules=[foo_build.ffibuilder.distutils_extension()], + ) + + For Setuptools (out-of-line only, but works in ABI or API mode; + recommended): + + .. code-block:: python + + # setup.py (with automatic dependency tracking) + from setuptools import setup + + setup( + ..., + setup_requires=["cffi>=1.0.0"], + cffi_modules=["package/foo_build.py:ffibuilder"], + install_requires=["cffi>=1.0.0"], + ) + + Note again that the ``foo_build.py`` example contains the following + lines, which mean that the ``ffibuilder`` is not actually compiled + when ``package.foo_build`` is merely imported---it will be compiled + independently by the Setuptools logic, using compilation parameters + provided by Setuptools: + + .. code-block:: python + + if __name__ == "__main__": # not when running with setuptools + ffibuilder.compile(verbose=True) + +* Note that some bundler tools that try to find all modules used by a + project, like PyInstaller, will miss ``_cffi_backend`` in the + out-of-line mode because your program contains no explicit ``import + cffi`` or ``import _cffi_backend``. You need to add + ``_cffi_backend`` explicitly (as a "hidden import" in PyInstaller, + but it can also be done more generally by adding the line ``import + _cffi_backend`` in your main program). + +Note that CFFI actually contains two different ``FFI`` classes. The +page `Using the ffi/lib objects`_ describes the common functionality. +It is what you get in the ``from package._foo import ffi`` lines above. +On the other hand, the extended ``FFI`` class is the one you get from +``import cffi; ffi_or_ffibuilder = cffi.FFI()``. It has the same +functionality (for in-line use), but also the extra methods described +below (to prepare the FFI). NOTE: We use the name ``ffibuilder`` +instead of ``ffi`` in the out-of-line context, when the code is about +producing a ``_foo.so`` file; this is an attempt to distinguish it +from the different ``ffi`` object that you get by later saying +``from _foo import ffi``. + +.. _`Using the ffi/lib objects`: using.html + +The reason for this split of functionality is that a regular program +using CFFI out-of-line does not need to import the ``cffi`` pure +Python package at all. (Internally it still needs ``_cffi_backend``, +a C extension module that comes with CFFI; this is why CFFI is also +listed in ``install_requires=..`` above. In the future this might be +split into a different PyPI package that only installs +``_cffi_backend``.) + +Note that a few small differences do exist: notably, ``from _foo import +ffi`` returns an object of a type written in C, which does not let you +add random attributes to it (nor does it have all the +underscore-prefixed internal attributes of the Python version). +Similarly, the ``lib`` objects returned by the C version are read-only, +apart from writes to global variables. Also, ``lib.__dict__`` does +not work before version 1.2 or if ``lib`` happens to declare a name +called ``__dict__`` (use instead ``dir(lib)``). The same is true +for ``lib.__class__``, ``lib.__all__`` and ``lib.__name__`` added +in successive versions. diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst index 15be27fa..6e028774 100644 --- a/doc/source/cdef.rst +++ b/doc/source/cdef.rst @@ -1,9 +1,18 @@ -====================================== -Preparing and Distributing modules -====================================== +========================= +Preparing Wrapper Modules +========================= .. contents:: +.. note:: + + A *wrapper module* is a Python module that uses CFFI to expose + functions and data from a C library, so that the rest of your + program can import it and call the library through the ``ffi`` and + ``lib`` objects. This page covers how to create wrapper modules. + See :ref:`buildtool_docs` for instructions on how to integrate with + a Python build backend and distribute wrapper modules. + There are three or four different ways to use CFFI in a project. In order of complexity: @@ -76,92 +85,6 @@ In order of complexity: # use ffi and lib here -.. _distutils-setuptools: - -* Finally, you can (but don't have to) use CFFI's **Distutils** or - **Setuptools integration** when writing a ``setup.py``. For - Distutils (only in out-of-line API mode; deprecated since - Python 3.10): - - .. code-block:: python - - # setup.py (requires CFFI to be installed first) - from distutils.core import setup - - import foo_build # possibly with sys.path tricks to find it - - setup( - ..., - ext_modules=[foo_build.ffibuilder.distutils_extension()], - ) - - For Setuptools (out-of-line only, but works in ABI or API mode; - recommended): - - .. code-block:: python - - # setup.py (with automatic dependency tracking) - from setuptools import setup - - setup( - ..., - setup_requires=["cffi>=1.0.0"], - cffi_modules=["package/foo_build.py:ffibuilder"], - install_requires=["cffi>=1.0.0"], - ) - - Note again that the ``foo_build.py`` example contains the following - lines, which mean that the ``ffibuilder`` is not actually compiled - when ``package.foo_build`` is merely imported---it will be compiled - independently by the Setuptools logic, using compilation parameters - provided by Setuptools: - - .. code-block:: python - - if __name__ == "__main__": # not when running with setuptools - ffibuilder.compile(verbose=True) - -* Note that some bundler tools that try to find all modules used by a - project, like PyInstaller, will miss ``_cffi_backend`` in the - out-of-line mode because your program contains no explicit ``import - cffi`` or ``import _cffi_backend``. You need to add - ``_cffi_backend`` explicitly (as a "hidden import" in PyInstaller, - but it can also be done more generally by adding the line ``import - _cffi_backend`` in your main program). - -Note that CFFI actually contains two different ``FFI`` classes. The -page `Using the ffi/lib objects`_ describes the common functionality. -It is what you get in the ``from package._foo import ffi`` lines above. -On the other hand, the extended ``FFI`` class is the one you get from -``import cffi; ffi_or_ffibuilder = cffi.FFI()``. It has the same -functionality (for in-line use), but also the extra methods described -below (to prepare the FFI). NOTE: We use the name ``ffibuilder`` -instead of ``ffi`` in the out-of-line context, when the code is about -producing a ``_foo.so`` file; this is an attempt to distinguish it -from the different ``ffi`` object that you get by later saying -``from _foo import ffi``. - -.. _`Using the ffi/lib objects`: using.html - -The reason for this split of functionality is that a regular program -using CFFI out-of-line does not need to import the ``cffi`` pure -Python package at all. (Internally it still needs ``_cffi_backend``, -a C extension module that comes with CFFI; this is why CFFI is also -listed in ``install_requires=..`` above. In the future this might be -split into a different PyPI package that only installs -``_cffi_backend``.) - -Note that a few small differences do exist: notably, ``from _foo import -ffi`` returns an object of a type written in C, which does not let you -add random attributes to it (nor does it have all the -underscore-prefixed internal attributes of the Python version). -Similarly, the ``lib`` objects returned by the C version are read-only, -apart from writes to global variables. Also, ``lib.__dict__`` does -not work before version 1.2 or if ``lib`` happens to declare a name -called ``__dict__`` (use instead ``dir(lib)``). The same is true -for ``lib.__class__``, ``lib.__all__`` and ``lib.__name__`` added -in successive versions. - .. _cdef: @@ -935,10 +858,10 @@ steps. and *if* the "stuff" part is big enough that import time is a concern, then rewrite it as described in `the out-of-line but still ABI mode`__ -above. Optionally, see also the `setuptools integration`__ paragraph. +above. Optionally, see also the :ref:`build backend and distribution +` documentation. .. __: out-of-line-abi_ -.. __: distutils-setuptools_ **API mode** if your CFFI project uses ``ffi.verify()``: @@ -951,16 +874,15 @@ above. Optionally, see also the `setuptools integration`__ paragraph. ffi.cdef("stuff") lib = ffi.verify("real C code") -then you should really rewrite it as described in `the out-of-line, -API mode`__ above. It avoids a number of issues that have caused +then you should really rewrite it as described in `the out-of-line, API +mode`__ above. It avoids a number of issues that have caused ``ffi.verify()`` to grow a number of extra arguments over time. Then -see the `distutils or setuptools`__ paragraph. Also, remember to -remove the ``ext_package=".."`` from your ``setup.py``, which was -sometimes needed with ``verify()`` but is just creating confusion with -``set_source()``. +see the :ref:`build backend and distribution ` +documentation. Also, remember to remove the ``ext_package=".."`` from +your ``setup.py``, which was sometimes needed with ``verify()`` but is +just creating confusion with ``set_source()``. .. __: out-of-line-api_ -.. __: distutils-setuptools_ The following example should work both with old (pre-1.0) and new versions of CFFI---supporting both is important to run on old diff --git a/doc/source/index.rst b/doc/source/index.rst index 54934f22..f5939d0e 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -16,4 +16,5 @@ copy-paste from header files or documentation. using ref cdef + buildtool embedding diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst index db9352d0..bb19e312 100644 --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -10,8 +10,16 @@ v2.0.0.dev0 with the limited API, so you must set py_limited_api=False when building extensions for the free-threaded build. * Added support for Python 3.14. (`#177`_) +* Added the ``gen-cffi-src`` command-line tool (also invocable as + ``python -m cffi.buildtool``), which lets build backends such as + `meson-python`_ generate CFFI extension C source without depending + on setuptools. See :doc:`buildtool` for details. Integrated from the + `cffi-buildtool`_ project by Rose Davidson. * WIP +.. _`meson-python`: https://meson-python.readthedocs.io/ +.. _`cffi-buildtool`: https://github.com/inklesspen/cffi-buildtool + .. _`#177`: https://github.com/python-cffi/cffi/pull/177 .. _`#178`: https://github.com/python-cffi/cffi/pull/178 diff --git a/pyproject.toml b/pyproject.toml index 447621fb..33ab91ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,9 @@ maintainers = [ [project.entry-points."distutils.setup_keywords"] cffi_modules = "cffi.setuptools_ext:cffi_modules" +[project.scripts] +gen-cffi-src = "cffi._buildtool:run" + [project.urls] Documentation = "https://cffi.readthedocs.io/" Changelog = "https://cffi.readthedocs.io/en/latest/whatsnew.html" diff --git a/src/cffi/_buildtool.py b/src/cffi/_buildtool.py new file mode 100644 index 00000000..85e3f34d --- /dev/null +++ b/src/cffi/_buildtool.py @@ -0,0 +1,197 @@ +# Integrated from the cffi-buildtool project by Rose Davidson +# (https://github.com/inklesspen/cffi-buildtool), under the following +# license: +# +# MIT License +# +# Copyright (c) 2024, Rose Davidson +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice (including the +# next paragraph) shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Implementation of the ``gen-cffi-src`` command-line tool. + +This module is private; the command line is the only supported +interface. Two subcommands: + +``exec-python`` + Execute a Python build script that constructs a :class:`cffi.FFI` + (the same kind of script that the CFFI docs' "Main mode of usage" + describes) and emit the generated C source. + +``read-sources`` + Build the :class:`cffi.FFI` from a separate ``cdef`` file and C + source prelude, then emit the generated C source. +""" + +import argparse +import io +import os +import sys + +from .api import FFI + + +def _execfile(pysrc, filename, globs): + compiled = compile(source=pysrc, filename=filename, mode='exec') + exec(compiled, globs, globs) + + +def find_ffi_in_python_script(pysrc, filename, ffivar): + """Execute ``pysrc`` and return the :class:`FFI` object it defines. + + The script is executed with ``__name__`` set to ``"cffi.buildtool"``, + so a trailing ``if __name__ == "__main__": ffibuilder.compile()`` + block in the script is skipped. + + ``ffivar`` is the name bound by the script to the :class:`FFI` + object, or to a callable that returns one. + + Raises :class:`NameError` if the name is not bound by the script, + or :class:`TypeError` if the name does not resolve to an + :class:`FFI` instance. + """ + globs = {'__name__': 'cffi.buildtool'} + _execfile(pysrc, filename, globs) + if ffivar not in globs: + raise NameError( + "Expected to find the FFI object with the name %r, " + "but it was not found." % (ffivar,) + ) + ffi = globs[ffivar] + if not isinstance(ffi, FFI) and callable(ffi): + # Maybe it's a callable that returns a FFI + ffi = ffi() + if not isinstance(ffi, FFI): + raise TypeError( + "Found an object with the name %r but it was not an " + "instance of cffi.api.FFI" % (ffivar,) + ) + return ffi + + +def make_ffi_from_sources(modulename, cdef, csrc): + """Build an :class:`FFI` from ``cdef`` text and a C source prelude.""" + ffibuilder = FFI() + ffibuilder.cdef(cdef) + ffibuilder.set_source(modulename, csrc) + return ffibuilder + + +def generate_c_source(ffi): + """Return the C source that :meth:`FFI.emit_c_code` would write.""" + output = io.StringIO() + ffi.emit_c_code(output) + return output.getvalue() + + +def exec_python(*, output, pyfile, ffi_var): + with pyfile: + ffi = find_ffi_in_python_script(pyfile.read(), pyfile.name, ffi_var) + generated = generate_c_source(ffi) + with output: + output.write(generated) + + +def read_sources(*, output, module_name, cdef_input, csrc_input): + with csrc_input, cdef_input: + csrc = csrc_input.read() + cdef = cdef_input.read() + ffi = make_ffi_from_sources(module_name, cdef, csrc) + generated = generate_c_source(ffi) + with output: + output.write(generated) + + +def _prog(): + # The same parser serves both documented invocations; make --help + # and usage messages show the one that was actually used. + argv0 = os.path.basename(sys.argv[0]) if sys.argv else '' + if argv0.startswith('gen-cffi-src'): + return 'gen-cffi-src' + return 'python -m cffi.buildtool' + + +parser = argparse.ArgumentParser( + prog=_prog(), + description='Generate CFFI C source for a build backend (e.g. meson-python).', +) +subparsers = parser.add_subparsers(dest='mode') + +exec_python_parser = subparsers.add_parser( + 'exec-python', + help='Execute a Python script to build an FFI object', +) +exec_python_parser.add_argument( + '--ffi-var', + default='ffibuilder', + help="Name of the FFI object in the Python script; defaults to 'ffibuilder'.", +) +exec_python_parser.add_argument( + 'pyfile', + type=argparse.FileType('r', encoding='utf-8'), + help='Path to the Python script', +) +exec_python_parser.add_argument( + 'output', + type=argparse.FileType('w', encoding='utf-8'), + help='Output path for the C source', +) + +read_sources_parser = subparsers.add_parser( + 'read-sources', + help='Read cdef and C source prelude files to build an FFI object', +) +read_sources_parser.add_argument( + 'module_name', + help='Full name of the generated module, including packages', +) +read_sources_parser.add_argument( + 'cdef', + type=argparse.FileType('r', encoding='utf-8'), + help='File containing C definitions', +) +read_sources_parser.add_argument( + 'csrc', + type=argparse.FileType('r', encoding='utf-8'), + help='File containing C source prelude', +) +read_sources_parser.add_argument( + 'output', + type=argparse.FileType('w', encoding='utf-8'), + help='Output path for the C source', +) + + +def run(args=None): + args = parser.parse_args(args=args) + if args.mode == 'exec-python': + exec_python(output=args.output, pyfile=args.pyfile, ffi_var=args.ffi_var) + elif args.mode == 'read-sources': + if args.cdef is args.csrc: + parser.error('cdef and csrc are the same file and should not be') + read_sources( + output=args.output, + module_name=args.module_name, + cdef_input=args.cdef, + csrc_input=args.csrc, + ) + else: + parser.error('a subcommand is required: exec-python or read-sources') + parser.exit(0) diff --git a/src/cffi/buildtool.py b/src/cffi/buildtool.py new file mode 100644 index 00000000..3e01ee14 --- /dev/null +++ b/src/cffi/buildtool.py @@ -0,0 +1,8 @@ +"""Entry point so ``python -m cffi.buildtool`` works. + +The implementation is private; see ``cffi/_buildtool.py``. +""" + +if __name__ == '__main__': + from cffi._buildtool import run + run() \ No newline at end of file diff --git a/testing/cffi1/buildtool_examples/build_script_example/meson.build b/testing/cffi1/buildtool_examples/build_script_example/meson.build new file mode 100644 index 00000000..b0e249ef --- /dev/null +++ b/testing/cffi1/buildtool_examples/build_script_example/meson.build @@ -0,0 +1,42 @@ +project( + 'squared', + 'c', + version: '0.1.0', + default_options: ['warning_level=2'], +) + +py = import('python').find_installation(pure: false) + +install_subdir('src/squared', install_dir: py.get_install_dir()) + +gen_cffi_src = find_program('gen-cffi-src') + +square_lib = static_library( + 'square', + 'src/csrc/square.c', + include_directories: include_directories('src/csrc'), +) +square_dep = declare_dependency( + link_with: square_lib, + include_directories: include_directories('src/csrc'), +) + +squared_ext_src = custom_target( + 'squared-cffi-src', + command: [ + gen_cffi_src, + 'exec-python', + '@INPUT@', + '@OUTPUT@', + ], + output: '_squared.c', + input: ['src/squared/_squared_build.py'], +) + +py.extension_module( + '_squared', + squared_ext_src, + subdir: 'squared', + install: true, + dependencies: [square_dep], +) diff --git a/testing/cffi1/buildtool_examples/build_script_example/pyproject.toml b/testing/cffi1/buildtool_examples/build_script_example/pyproject.toml new file mode 120000 index 00000000..d6064133 --- /dev/null +++ b/testing/cffi1/buildtool_examples/build_script_example/pyproject.toml @@ -0,0 +1 @@ +../common/pyproject.toml \ No newline at end of file diff --git a/testing/cffi1/buildtool_examples/build_script_example/src/csrc/square.c b/testing/cffi1/buildtool_examples/build_script_example/src/csrc/square.c new file mode 120000 index 00000000..ae7ce3e6 --- /dev/null +++ b/testing/cffi1/buildtool_examples/build_script_example/src/csrc/square.c @@ -0,0 +1 @@ +../../../common/src/csrc/square.c \ No newline at end of file diff --git a/testing/cffi1/buildtool_examples/build_script_example/src/csrc/square.h b/testing/cffi1/buildtool_examples/build_script_example/src/csrc/square.h new file mode 120000 index 00000000..fb6c63f2 --- /dev/null +++ b/testing/cffi1/buildtool_examples/build_script_example/src/csrc/square.h @@ -0,0 +1 @@ +../../../common/src/csrc/square.h \ No newline at end of file diff --git a/testing/cffi1/buildtool_examples/build_script_example/src/squared/__init__.py b/testing/cffi1/buildtool_examples/build_script_example/src/squared/__init__.py new file mode 120000 index 00000000..12d4d27e --- /dev/null +++ b/testing/cffi1/buildtool_examples/build_script_example/src/squared/__init__.py @@ -0,0 +1 @@ +../../../common/src/squared/__init__.py \ No newline at end of file diff --git a/testing/cffi1/buildtool_examples/build_script_example/src/squared/_squared_build.py b/testing/cffi1/buildtool_examples/build_script_example/src/squared/_squared_build.py new file mode 100644 index 00000000..f44e92c5 --- /dev/null +++ b/testing/cffi1/buildtool_examples/build_script_example/src/squared/_squared_build.py @@ -0,0 +1,13 @@ +from cffi import FFI + +ffibuilder = FFI() + +ffibuilder.cdef("int square(int n);") + +ffibuilder.set_source( + "squared._squared", + '#include "square.h"', +) + +if __name__ == "__main__": + ffibuilder.compile(verbose=True) diff --git a/testing/cffi1/buildtool_examples/cdef_example/meson.build b/testing/cffi1/buildtool_examples/cdef_example/meson.build new file mode 100644 index 00000000..b9486b17 --- /dev/null +++ b/testing/cffi1/buildtool_examples/cdef_example/meson.build @@ -0,0 +1,44 @@ +project( + 'squared', + 'c', + version: '0.1.0', + default_options: ['warning_level=2'], +) + +py = import('python').find_installation(pure: false) + +install_subdir('src/squared', install_dir: py.get_install_dir()) + +gen_cffi_src = find_program('gen-cffi-src') + +square_lib = static_library( + 'square', + 'src/csrc/square.c', + include_directories: include_directories('src/csrc'), +) +square_dep = declare_dependency( + link_with: square_lib, + include_directories: include_directories('src/csrc'), +) + +squared_ext_src = custom_target( + 'squared-cffi-src', + command: [ + gen_cffi_src, + 'read-sources', + 'squared._squared', + '@INPUT0@', + '@INPUT1@', + '@OUTPUT@', + ], + output: '_squared.c', + input: ['src/squared/squared.cdef.txt', 'src/squared/squared.csrc.c'], +) + +py.extension_module( + '_squared', + squared_ext_src, + subdir: 'squared', + install: true, + dependencies: [square_dep], +) diff --git a/testing/cffi1/buildtool_examples/cdef_example/pyproject.toml b/testing/cffi1/buildtool_examples/cdef_example/pyproject.toml new file mode 120000 index 00000000..d6064133 --- /dev/null +++ b/testing/cffi1/buildtool_examples/cdef_example/pyproject.toml @@ -0,0 +1 @@ +../common/pyproject.toml \ No newline at end of file diff --git a/testing/cffi1/buildtool_examples/cdef_example/src/csrc/square.c b/testing/cffi1/buildtool_examples/cdef_example/src/csrc/square.c new file mode 120000 index 00000000..ae7ce3e6 --- /dev/null +++ b/testing/cffi1/buildtool_examples/cdef_example/src/csrc/square.c @@ -0,0 +1 @@ +../../../common/src/csrc/square.c \ No newline at end of file diff --git a/testing/cffi1/buildtool_examples/cdef_example/src/csrc/square.h b/testing/cffi1/buildtool_examples/cdef_example/src/csrc/square.h new file mode 120000 index 00000000..fb6c63f2 --- /dev/null +++ b/testing/cffi1/buildtool_examples/cdef_example/src/csrc/square.h @@ -0,0 +1 @@ +../../../common/src/csrc/square.h \ No newline at end of file diff --git a/testing/cffi1/buildtool_examples/cdef_example/src/squared/__init__.py b/testing/cffi1/buildtool_examples/cdef_example/src/squared/__init__.py new file mode 120000 index 00000000..12d4d27e --- /dev/null +++ b/testing/cffi1/buildtool_examples/cdef_example/src/squared/__init__.py @@ -0,0 +1 @@ +../../../common/src/squared/__init__.py \ No newline at end of file diff --git a/testing/cffi1/buildtool_examples/cdef_example/src/squared/squared.cdef.txt b/testing/cffi1/buildtool_examples/cdef_example/src/squared/squared.cdef.txt new file mode 100644 index 00000000..72afcbd7 --- /dev/null +++ b/testing/cffi1/buildtool_examples/cdef_example/src/squared/squared.cdef.txt @@ -0,0 +1 @@ +int square(int n); \ No newline at end of file diff --git a/testing/cffi1/buildtool_examples/cdef_example/src/squared/squared.csrc.c b/testing/cffi1/buildtool_examples/cdef_example/src/squared/squared.csrc.c new file mode 100644 index 00000000..930ea7ee --- /dev/null +++ b/testing/cffi1/buildtool_examples/cdef_example/src/squared/squared.csrc.c @@ -0,0 +1 @@ +#include "square.h" diff --git a/testing/cffi1/buildtool_examples/common/pyproject.toml b/testing/cffi1/buildtool_examples/common/pyproject.toml new file mode 100644 index 00000000..45b21900 --- /dev/null +++ b/testing/cffi1/buildtool_examples/common/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +build-backend = 'mesonpy' +requires = ['meson-python', 'cffi'] + +[project] +name = 'squared' +version = '0.1.0' +description = 'Small self-contained example project that builds a CFFI extension via meson-python.' +requires-python = '>=3.9' +dependencies = ['cffi'] diff --git a/testing/cffi1/buildtool_examples/common/src/csrc/square.c b/testing/cffi1/buildtool_examples/common/src/csrc/square.c new file mode 100644 index 00000000..05d6092f --- /dev/null +++ b/testing/cffi1/buildtool_examples/common/src/csrc/square.c @@ -0,0 +1,5 @@ +#include "square.h" + +int square(int n) { + return n * n; +} diff --git a/testing/cffi1/buildtool_examples/common/src/csrc/square.h b/testing/cffi1/buildtool_examples/common/src/csrc/square.h new file mode 100644 index 00000000..356a5771 --- /dev/null +++ b/testing/cffi1/buildtool_examples/common/src/csrc/square.h @@ -0,0 +1,6 @@ +#ifndef SQUARE_H +#define SQUARE_H + +int square(int n); + +#endif diff --git a/testing/cffi1/buildtool_examples/common/src/squared/__init__.py b/testing/cffi1/buildtool_examples/common/src/squared/__init__.py new file mode 100644 index 00000000..156e67d5 --- /dev/null +++ b/testing/cffi1/buildtool_examples/common/src/squared/__init__.py @@ -0,0 +1,5 @@ +from ._squared import ffi, lib + + +def squared(n): + return lib.square(n) diff --git a/testing/cffi1/test_buildtool.py b/testing/cffi1/test_buildtool.py new file mode 100644 index 00000000..72f08417 --- /dev/null +++ b/testing/cffi1/test_buildtool.py @@ -0,0 +1,186 @@ +"""Tests for the buildtool command-line tool. + +The command line is the buildtool's only public interface, so these +tests drive it exclusively through subprocesses. +""" + +import os +import shutil +import subprocess +import sys +import sysconfig + +import pytest + +pytestmark = [ + pytest.mark.thread_unsafe(reason="spawns subprocesses, slow"), +] + + +SIMPLE_SCRIPT = """\ +from cffi import FFI + +ffibuilder = FFI() + +ffibuilder.cdef("int square(int n);") + +ffibuilder.set_source("squared._squared", '#include "square.h"') + +something_else = 42 + +if __name__ == "__main__": + ffibuilder.compile(verbose=True) +""" + +CALLABLE_SCRIPT = """\ +from cffi import FFI + +def make_ffi(): + ffibuilder = FFI() + ffibuilder.cdef("int square(int n);") + ffibuilder.set_source("squared._squared", '#include "square.h"') + return ffibuilder + +def something_else(): + return 42 +""" + + +def _gen_cffi_src_path(): + """Locate the installed ``gen-cffi-src`` script, or return None.""" + exe = "gen-cffi-src" + (".exe" if sys.platform == "win32" else "") + candidate = os.path.join(sysconfig.get_path("scripts"), exe) + if os.path.exists(candidate): + return candidate + return shutil.which("gen-cffi-src") + + +def _run(argv, *args): + return subprocess.run( + [*argv, *args], + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + ) + + +# `gen-cffi-src` can also be invoked via `python -m cffi.buildtool`. +# This fixture enables testing both invocations. +@pytest.fixture(params=["module", "script"]) +def run_buildtool(request): + """Run the buildtool CLI in a subprocess, via both invocations.""" + if request.param == "script": + script = _gen_cffi_src_path() + if script is None: + pytest.skip("the gen-cffi-src script is not installed") + argv = [script] + else: + argv = [sys.executable, "-m", "cffi.buildtool"] + + def run(*args): + return _run(argv, *args) + + return run + + +def test_exec_python(tmp_path, run_buildtool): + pyfile = tmp_path / "_squared_build.py" + pyfile.write_text(SIMPLE_SCRIPT) + output = tmp_path / "out.c" + proc = run_buildtool("exec-python", str(pyfile), str(output)) + assert proc.returncode == 0, proc.stderr + generated = output.read_text() + assert "square" in generated + # sanity check: the emitted C source is the thing meson-python will compile + assert "PyInit" in generated or "_cffi_f_" in generated + + +def test_exec_python_ffi_var(tmp_path, run_buildtool): + pyfile = tmp_path / "_squared_build.py" + pyfile.write_text(CALLABLE_SCRIPT) + output = tmp_path / "out.c" + proc = run_buildtool( + "exec-python", "--ffi-var", "make_ffi", str(pyfile), str(output) + ) + assert proc.returncode == 0, proc.stderr + assert "square" in output.read_text() + + +def test_exec_python_name_not_found(tmp_path, run_buildtool): + pyfile = tmp_path / "_squared_build.py" + pyfile.write_text(SIMPLE_SCRIPT) + output = tmp_path / "out.c" + proc = run_buildtool( + "exec-python", "--ffi-var", "notfound", str(pyfile), str(output) + ) + assert proc.returncode != 0 + assert "NameError" in proc.stderr + assert "'notfound'" in proc.stderr + + +@pytest.mark.parametrize( + "script", [SIMPLE_SCRIPT, CALLABLE_SCRIPT], ids=["simple", "callable"] +) +def test_exec_python_wrong_type(tmp_path, run_buildtool, script): + pyfile = tmp_path / "_squared_build.py" + pyfile.write_text(script) + output = tmp_path / "out.c" + proc = run_buildtool( + "exec-python", "--ffi-var", "something_else", str(pyfile), str(output) + ) + assert proc.returncode != 0 + assert "not an instance of cffi.api.FFI" in proc.stderr + + +def test_read_sources(tmp_path, run_buildtool): + cdef = tmp_path / "squared.cdef.txt" + cdef.write_text("int square(int n);\n") + csrc = tmp_path / "squared.csrc.c" + csrc.write_text('#include "square.h"\n') + output = tmp_path / "out.c" + proc = run_buildtool( + "read-sources", "squared._squared", str(cdef), str(csrc), str(output) + ) + assert proc.returncode == 0, proc.stderr + generated = output.read_text() + assert "square" in generated + assert "PyInit" in generated or "_cffi_f_" in generated + + +def test_read_sources_same_input_fails(run_buildtool): + proc = run_buildtool("read-sources", "_squared", "-", "-", "-") + assert proc.returncode != 0 + assert "are the same file and should not be" in proc.stderr + + +def test_no_subcommand(run_buildtool): + proc = run_buildtool() + assert proc.returncode != 0 + assert "a subcommand is required" in proc.stderr + + +def test_module_help_names_module_invocation(): + proc = _run([sys.executable, "-m", "cffi.buildtool"], "--help") + assert proc.returncode == 0, proc.stderr + assert proc.stdout.startswith("usage: python -m cffi.buildtool") + + +def test_script_help_names_script_invocation(): + script = _gen_cffi_src_path() + if script is None: + pytest.skip("the gen-cffi-src script is not installed") + proc = _run([script], "--help") + assert proc.returncode == 0, proc.stderr + assert proc.stdout.startswith("usage: gen-cffi-src") + + +def test_import_is_inert(): + # importing the entry-point module must not run the CLI + proc = subprocess.run( + [sys.executable, "-c", "import cffi.buildtool"], + capture_output=True, + text=True, + ) + assert proc.returncode == 0, proc.stderr + assert proc.stdout == "" + assert proc.stderr == "" diff --git a/testing/cffi1/test_buildtool_meson.py b/testing/cffi1/test_buildtool_meson.py new file mode 100644 index 00000000..8f2937be --- /dev/null +++ b/testing/cffi1/test_buildtool_meson.py @@ -0,0 +1,88 @@ +"""End-to-end test: build a self-contained CFFI extension with meson-python. + +The test provisions a fresh nested venv under ``tmp_path`` using the +stdlib :mod:`venv` module, installs ``cffi`` (from the current source +tree) and ``meson-python`` into it, installs one of the small example +projects that live under ``testing/cffi1/buildtool_examples/``, and then +imports the built extension to confirm it works. + +""" + +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +import cffi + +pytestmark = [ + pytest.mark.thread_unsafe(reason="spawns subprocesses, slow"), +] + +try: + import mesonpy +except ImportError: + pytest.skip("Test requires meson-python", allow_module_level=True) + + +HERE = Path(__file__).resolve().parent +EXAMPLE_PROJECT = HERE / "buildtool_examples" / "build_script_example" +EXAMPLE_PROJECT2 = HERE / "buildtool_examples" / "cdef_example" +CFFI_DIR = HERE.parent.parent + + +def _venv_python(venv_dir): + if sys.platform == "win32": + return venv_dir / "Scripts" / "python.exe" + return venv_dir / "bin" / "python" + + +@pytest.mark.parametrize("project", [EXAMPLE_PROJECT, EXAMPLE_PROJECT2]) +def test_meson_python_build(tmp_path, project): + venv_dir = tmp_path / "venv" + subprocess.check_call([sys.executable, "-m", "venv", str(venv_dir)]) + venv_python = _venv_python(venv_dir) + assert venv_python.exists(), venv_python + + # Upgrade pip so --no-build-isolation behaves consistently with recent + # resolver behaviour on older base images. + subprocess.check_call([ + str(venv_python), "-m", "pip", "install", "--upgrade", "pip", + ]) + + # Install build-time deps into the nested venv. + subprocess.check_call([ + str(venv_python), "-m", "pip", "install", "meson-python", CFFI_DIR + ]) + + # Copy the example project so nothing is written back into the + # source tree + project_dir = tmp_path / "project" + shutil.copytree(project, project_dir) + + # The example meson.build files locate the codegen tool with + # find_program('gen-cffi-src'), which searches PATH. pip only puts + # an environment's scripts directory on PATH for isolated builds, + # so with --no-build-isolation the nested venv's script must be + # made findable by hand. + env = os.environ.copy() + env["PATH"] = str(venv_python.parent) + os.pathsep + env.get("PATH", "") + + # --no-build-isolation to ensure the test runs against the CFFI build we want to test + proc = subprocess.run([ + str(venv_python), "-m", "pip", "install", "-v", + "--no-build-isolation", str(project_dir), + ], env=env, capture_output=True, text=True) + assert proc.returncode == 0, proc.stdout + proc.stderr + + # Confirm the built extension imports and behaves as expected. + subprocess.check_call([ + str(venv_python), "-c", + "from squared import squared; " + "assert squared(7) == 49; " + "assert squared(-3) == 9", + ])