diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 2457650df..42f2869cd 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -355,6 +355,12 @@ jobs: run: >- make clean shell: bash + - name: Pre-install ccache for gcov [FIXME] + if: steps.build_type.outputs.build_type == 'source' + run: >- + sudo apt update -y + && sudo apt install -y ccache + shell: bash - name: Self-install (from ${{ steps.build_type.outputs.build_type }}) env: MULTIDICT_NO_EXTENSIONS: ${{ matrix.no-extensions }} @@ -365,12 +371,56 @@ jobs: && '.' || steps.wheel-file.outputs.path }}' + - name: Experimental display of multidict files [FIXME] + run: pip show multidict --files - name: Run unittests run: >- python -Im pytest tests -v --cov-report xml --junitxml=.test-results/pytest/test.xml --${{ matrix.no-extensions == 'Y' && 'no-' || '' }}c-extensions + - name: Experimentally find gcda/gcno [FIXME] + if: >- + !cancelled() + && runner.os == 'Linux' + run: | + find . -name '*.gc*' + - name: Experimentally find gcda/gcno in site-packages [FIXME] + if: >- + !cancelled() + && runner.os == 'Linux' + run: | + find "$(python -Ic 'import importlib.resources; print(str(importlib.resources.files("multidict")))')" -name '*.gc*' + - name: Experimentally install Gcovr [FIXME] + if: >- + !cancelled() + && runner.os == 'Linux' + run: | + pip install gcovr + - name: Experimentally make a coverage dir [FIXME] + if: >- + !cancelled() + && runner.os == 'Linux' + run: | + mkdir -pv coverage + - name: Experimentally run Gcovr [FIXME] + if: >- + !cancelled() + && runner.os == 'Linux' + run: | + gcovr --verbose + - name: See Gcovr leftovers at home [FIXME] + if: >- + !cancelled() + && runner.os == 'Linux' + run: | + find ~/ -type f -name '*.gcov.json.gz' + - name: See Gcovr leftovers in workdir [FIXME] + if: >- + !cancelled() + && runner.os == 'Linux' + run: | + find ../.. -type f -name '*.gcov.json.gz' - name: Produce markdown test summary from JUnit if: >- !cancelled() diff --git a/MANIFEST.in b/MANIFEST.in index 43625782e..62e6ba28d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include .coveragerc +include gcovr.cfg include pyproject.toml include pytest.ini include LICENSE diff --git a/gcovr.cfg b/gcovr.cfg new file mode 100644 index 000000000..e46703e34 --- /dev/null +++ b/gcovr.cfg @@ -0,0 +1,10 @@ +filter = multidict/ + +html-details = coverage/index.html + +print-summary = yes + +search-path = multidict/__tracing-data__/multidict/_multidict.gcda +search-path = multidict/__tracing-data__/multidict/_multidict.gcno + +xml = coverage/coverage.xml diff --git a/pyproject.toml b/pyproject.toml index cf4a8b354..730bd527c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,4 +13,7 @@ enable = ["cpython-freethreading"] [tool.cibuildwheel.linux] # Re-enable 32-bit builds (disabled by default in cibuildwheel 3.0) archs = ["auto", "auto32"] -before-all = "yum install -y libffi-devel || apk add --upgrade libffi-dev || apt-get install libffi-dev" +before-all = "yum install -y ccache libffi-devel || apk add --upgrade ccache libffi-dev || apt-get install ccache libffi-dev" + +[tool.cibuildwheel.linux.environment] +MULTIDICT_DEBUG_BUILD = "1" diff --git a/setup.py b/setup.py index 318229fcc..ccf29d231 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,11 @@ +import functools +import json import os +import pathlib import platform import sys +from distutils import log from setuptools import Extension, setup NO_EXTENSIONS = bool(os.environ.get("MULTIDICT_NO_EXTENSIONS")) @@ -11,8 +15,27 @@ NO_EXTENSIONS = True CFLAGS = ["-O0", "-g3", "-UNDEBUG"] if DEBUG_BUILD else ["-O3", "-DNDEBUG"] +# https://gcc.gnu.org/onlinedocs/gcc/Gcov-Data-Files.html +# `-ftest-coverage` -> `.gcno` is in the same place as `.o` -> `{self.build_temp}/multidict` +# `-fprofile-dir` -> change the location of the `.gcda` file +# +# https://gcc.gnu.org/onlinedocs/gcc/Invoking-Gcov.html +# `-fkeep-inline-functions` +# `-fkeep-static-functions` +# `-fprofile-prefix-map=old=new` +# `--coverage` is an alias for `-fprofile-arcs -ftest-coverage` when compiling and `-lgcov` when linking; `-coverage` seems to be its synonym. +# `-fprofile-abs-path` makes `gcovr` more robust — https://gcovr.com/en/stable/guide/compiling.html#compiler-options +if DEBUG_BUILD: + CFLAGS.extend([ + '--coverage', + '-fkeep-inline-functions', + '-fkeep-static-functions', + '-fprofile-abs-path', + ]) -if platform.system() != "Windows": +LDFLAGS = ['--coverage'] if DEBUG_BUILD else [] + +if platform.system() != "Windows" and False: CFLAGS.extend( [ "-std=c11", @@ -25,20 +48,201 @@ ] ) +if DEBUG_BUILD: + # https://gcovr.com/en/stable/cookbook.html#how-to-collect-coverage-for-c-extensions-in-python + os.environ['CC'] = 'ccache gcc' # no distutils/setuptools equivalent + extensions = [ Extension( "multidict._multidict", ["multidict/_multidict.c"], extra_compile_args=CFLAGS, + extra_link_args=LDFLAGS, ), ] +from setuptools.command.build_ext import build_ext + + +class TraceableBinaryExtensionCmd(build_ext): + def _ext_mod_path_for(self, ext) -> list[str]: + if not DEBUG_BUILD: + raise LookupError + + fullname = self.get_ext_fullname(ext.name) + return fullname.split('.') + + def _ext_tracing_file_for(self, ext) -> pathlib.Path: + if not DEBUG_BUILD: + raise LookupError + + modpath = self._ext_mod_path_for(ext) + + # NOTE: `.o` file is unnecessary for gcovr to function + return pathlib.Path(*modpath).with_suffix('.gcno') + + def _ext_tracing_data_dir_for(self, ext) -> tuple[str]: + if not DEBUG_BUILD: + raise LookupError + + modpath = self._ext_mod_path_for(ext) + package = '.'.join(modpath[:-1]) + build_py = self.get_finalized_command('build_py') # ?? + package_dir = pathlib.Path(build_py.get_package_dir(package)) + return package_dir / '__tracing-data__' + + @functools.cached_property + def _ext_tracing_data_dir_map(self) -> dict[str, tuple[str]]: + extensions_with_tracing_data = self.extensions if DEBUG_BUILD else () + + return { + ext.name: self._ext_tracing_data_dir_for(ext) + for ext in extensions_with_tracing_data + } + + def _extra_ext_data_files_for(self, ext) -> tuple[str]: + if not DEBUG_BUILD: + return () + + tracing_data_in_package_dir = self._ext_tracing_data_dir_map[ext.name] + tracing_data_file_in_tmp_dir = self._ext_tracing_file_for(ext) + tracing_data_file_in_package_dir = tracing_data_in_package_dir / tracing_data_file_in_tmp_dir + build_meta_json_path = tracing_data_in_package_dir / 'build-metadata.json' + + return (build_meta_json_path, tracing_data_file_in_package_dir) + + @functools.cached_property + def _extra_ext_data_files_map(self) -> dict[str, tuple[str]]: + extensions_with_tracing_data = self.extensions if DEBUG_BUILD else () + + return { + ext.name: self._extra_ext_data_files_for(ext) + for ext in extensions_with_tracing_data + } + + @functools.cached_property + def _extra_wheel_data_files(self) -> tuple[pathlib.Path]: + extensions_with_tracing_data = self.extensions if DEBUG_BUILD else () + + return tuple( + relative_path + for ext in extensions_with_tracing_data + for relative_path in self._extra_ext_data_files_map[ext.name] + ) + + def get_outputs(self) -> list[str]: + """Return absolute file paths to be included in the wheel.""" + base_outputs = super().get_outputs() + + build_dir_path = pathlib.Path(self.build_lib) + tracing_outputs = ( + build_dir_path / relative_path + for relative_path in self._extra_wheel_data_files + ) + + # NOTE: Files returned here end up in wheels and then `site-packages/`. + # NOTE: Editable installs rely on the tracing files to be copied into + # NOTE: the source checkout, which is happening in `run()`. + return [*base_outputs, *tracing_outputs] + + def build_extension(self, ext): + super().build_extension(ext) + + if not DEBUG_BUILD: + log.info('Not a debug build. Skipping tracing data...') + return + + log.info('Copying tracing data into the build directory') + tracing_data_file_in_tmp_dir = self._ext_tracing_file_for(ext) + tracing_data_in_package_dir = self._ext_tracing_data_dir_map[ext.name] + tracing_data_file_in_package_dir = tracing_data_in_package_dir / tracing_data_file_in_tmp_dir + + tracing_data_in_build_dir = pathlib.Path(self.build_lib) / tracing_data_in_package_dir + + tracing_data_file_in_build_dir_absolute = tracing_data_in_build_dir / tracing_data_file_in_tmp_dir + + tracing_data_file_in_build_dir_absolute.parent.mkdir(exist_ok=True, parents=True) + # NOTE: `gcc` writes `.gcno` files next to `.o` which is the temporary + # NOTE: directory for us (`build_temp`). This copies it over into the + # NOTE: regular build directory (`build_lib`) producing the layout we + # NOTE: expect in wheels / site-packages / source checkout. It's later + # NOTE: copied over to those places along with the shared libraries. + self.copy_file( + pathlib.Path(self.build_temp) / tracing_data_file_in_tmp_dir, + tracing_data_file_in_build_dir_absolute, + level=self.verbose, + ) + + # NOTE: `gcc` writes an absolute path to `.gcno` files into the shared + # NOTE: library at build time. It may point to an arbitrary temporary + # NOTE: directory that we don't care for. When pytest imports this + # NOTE: file, the C-extension attempts writing a `.gcda` file in the + # NOTE: same directory. We want it to be written in the source checkout + # NOTE: next to the tracing file. For this, we record information about + # NOTE: the desired location relative to the project root, bundle it + # NOTE: in the wheel and the tests will read from it, and set the + # NOTE: respective environment variables so the coverage data ends up + # NOTE: where we expect it to. `gcovr` will also work better with + # NOTE: predictable data locations. + # NOTE: + # NOTE: The contributors won't have to set the env vars manually + # NOTE: $ GCOV_PREFIX=multidict/__tracing-data__/ \ + # NOTE: GCOV_PREFIX_STRIP=2 \ + # NOTE: python -Im pytest + build_meta_path = tracing_data_in_build_dir / 'build-metadata.json' + tmp_path_length = len(pathlib.Path(self.build_temp).resolve().parents) + build_meta_path.write_text( + json.dumps( + { + 'GCOV_PREFIX': str(tracing_data_in_package_dir), + 'GCOV_PREFIX_STRIP': str(tmp_path_length), + }, + ), + encoding='utf-8', + ) + + def run(self): + super().run() + + if not self.inplace: + log.info('Not an editable install. Skipping tracing data...') + + if not DEBUG_BUILD: + log.info('Not a debug build. Skipping tracing data...') + return + + log.info( + 'Editable install in debug mode. Copying tracing data in-tree...', + ) + + # NOTE: Editable installs usually expect data in-tree. This handles + # NOTE: the cases of `python setup.py build_ext --inplace` and + # NOTE: `pip install -e .` + # NOTE: Normal wheel builds include files returned by `get_outputs()`. + + build_dir_path = pathlib.Path(self.build_lib) + for relative_path in self._extra_wheel_data_files: + relative_path.parent.mkdir(exist_ok=True, parents=True) + self.copy_file( + build_dir_path / relative_path, + relative_path, + level=self.verbose, + ) + + for ext in self.extensions: + tracing_data_in_pkg_dir = self._ext_tracing_data_dir_map[ext.name] + (tracing_data_in_pkg_dir / '.gitignore').write_text('*\n') + + if not NO_EXTENSIONS: print("*********************") print("* Accelerated build *") print("*********************") - setup(ext_modules=extensions) + setup( + cmdclass={'build_ext': TraceableBinaryExtensionCmd}, + ext_modules=extensions, + ) else: print("*********************") print("* Pure Python build *") diff --git a/tests/conftest.py b/tests/conftest.py index dfc1a6703..a00efba7e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,15 @@ from __future__ import annotations import argparse +import importlib.resources +import json +import os import pickle from dataclasses import dataclass from functools import cached_property from importlib import import_module +from pathlib import Path +from shutil import copytree from types import ModuleType from typing import Callable, Type, Union @@ -68,11 +73,47 @@ def multidict_implementation(request: pytest.FixtureRequest) -> MultidictImpleme return request.param # type: ignore[no-any-return] +@pytest.fixture(scope="session") +def _gcov_configuration() -> None: + """Configure the C-extension gcda write location in debug mode.""" + # NOTE: This is not using `monkeypatch` because it's unavailable with the + # NOTE: session scope. Additionally, we need these environment variables + # NOTE: to survive until the module gets to writing data to disk. + # NOTE: We don't currently run with `pytest-xdist` but might have to + # NOTE: improve this if we start, to avoid data races. Also, should we + # NOTE: integrate `tmp_path` instead of writing to `site-packages/`? + + tracing_data_dir_path = ( + # FIXME: read from `pyproject.toml`? + importlib.resources.files('multidict') + / '__tracing-data__' + ) + if not tracing_data_dir_path.is_dir(): + # NOTE: The C-extention was probably compiled without tracing or + # NOTE: packaging is borked. + return + + project_root = Path(__file__).parent.parent.resolve() + + build_meta_json_path = tracing_data_dir_path / 'build-metadata.json' + gcov_env = json.loads(build_meta_json_path.read_text(encoding='utf-8')) + + in_project_tracing_data_dir_path = project_root / gcov_env['GCOV_PREFIX'] + gcov_env['GCOV_PREFIX'] = str(in_project_tracing_data_dir_path) + + copytree(tracing_data_dir_path, in_project_tracing_data_dir_path, dirs_exist_ok=True) + + os.environ.update(gcov_env) + + @pytest.fixture(scope="session") def multidict_module( multidict_implementation: MultidictImplementation, + request: pytest.FixtureRequest, ) -> ModuleType: """Return a pre-imported module containing a multidict variant.""" + if not multidict_implementation.is_pure_python: + request.getfixturevalue('_gcov_configuration') return multidict_implementation.imported_module