From 0035444abc6aca91c701d778e5eae1d8cbfff9c5 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Wed, 3 Dec 2025 16:28:55 +0000 Subject: [PATCH 1/6] Tweak test tolerances Tighten tolerance for float64 from 1e-3 ~ 1e-4 to 1e-9. Relax tolerance for float32 to 1e-2, as increasing the number of valid examples lead to the discovery of dotv: float32 rounding error is 10x of NumPy (#142). Swap actual <-> desired in assert_almost_equal, as the two are not symmetric. --- tests/test_dotv.py | 4 ++-- tests/test_gemm.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_dotv.py b/tests/test_dotv.py index eb97a7e8..8c671dca 100644 --- a/tests/test_dotv.py +++ b/tests/test_dotv.py @@ -19,7 +19,7 @@ def test_memoryview_double_noconj(A, B): assume(B is not None) numpy_result = A.dot(B) result = dotv(A, B) - assert_allclose([numpy_result], result, atol=1e-3, rtol=1e-3) + assert_allclose(result, numpy_result, atol=1e-9, rtol=1e-9) @given( @@ -35,4 +35,4 @@ def test_memoryview_float_noconj(A, B): assume(B is not None) numpy_result = A.dot(B) result = dotv(A, B) - assert_allclose([numpy_result], result, atol=1e-4, rtol=1e-3) + assert_allclose(result, numpy_result, atol=1e-2, rtol=1e-4) diff --git a/tests/test_gemm.py b/tests/test_gemm.py index 5c51c7d6..f244026b 100644 --- a/tests/test_gemm.py +++ b/tests/test_gemm.py @@ -64,7 +64,7 @@ def test_memoryview_double_notrans(A, B, a_rows, a_cols, out_cols): assume(C.size >= 1) gemm(A, B, out=C) numpy_result = A.dot(B) - assert_allclose(numpy_result, C, atol=1e-3, rtol=1e-3) + assert_allclose(C, numpy_result, atol=1e-9, rtol=1e-9) @given( @@ -84,4 +84,4 @@ def test_memoryview_float_notrans(A, B, a_rows, a_cols, out_cols): assume(C.size >= 1) gemm(A, B, out=C) numpy_result = A.dot(B) - assert_allclose(numpy_result, C, atol=1e-3, rtol=1e-3) + assert_allclose(C, numpy_result, atol=1e-2, rtol=1e-4) From 820905ccc4e0774b56c72aab6807c2b5ed54f83a Mon Sep 17 00:00:00 2001 From: crusaderky Date: Mon, 8 Dec 2025 17:44:28 +0000 Subject: [PATCH 2/6] Tweak strategies ranges Add test coverage for arrays of size 1. Remove unused, confusing default parameters from custom strategies. --- tests/blis_tests_common.py | 9 ++++----- tests/test_dotv.py | 8 ++++---- tests/test_gemm.py | 8 ++++---- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/blis_tests_common.py b/tests/blis_tests_common.py index e6ca2e7b..2f63bf80 100644 --- a/tests/blis_tests_common.py +++ b/tests/blis_tests_common.py @@ -7,11 +7,12 @@ from hypothesis.extra.numpy import arrays -def lengths(lo=1, hi=10): + +def lengths(lo, hi): return integers(min_value=lo, max_value=hi) -def ndarrays_of_shape(shape, lo=-1000.0, hi=1000.0, dtype="float64"): +def ndarrays_of_shape(shape, lo, hi, dtype): width = 64 if dtype == "float64" else 32 return arrays( dtype, @@ -24,9 +25,7 @@ def ndarrays_of_shape(shape, lo=-1000.0, hi=1000.0, dtype="float64"): ) -def ndarrays( - min_len=0, max_len=10, min_val=-10000000.0, max_val=1000000.0, dtype="float64" -): +def ndarrays(min_len, max_len, min_val, max_val, dtype): return lengths(lo=min_len, hi=max_len).flatmap( lambda n: ndarrays_of_shape(n, lo=min_val, hi=max_val, dtype=dtype) ) diff --git a/tests/test_dotv.py b/tests/test_dotv.py index 8c671dca..ba530e0c 100644 --- a/tests/test_dotv.py +++ b/tests/test_dotv.py @@ -7,8 +7,8 @@ @given( - ndarrays(min_len=10, max_len=100, min_val=-100.0, max_val=100.0, dtype="float64"), - ndarrays(min_len=10, max_len=100, min_val=-100.0, max_val=100.0, dtype="float64"), + ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float64"), + ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float64"), ) def test_memoryview_double_noconj(A, B): if len(A) < len(B): @@ -23,8 +23,8 @@ def test_memoryview_double_noconj(A, B): @given( - ndarrays(min_len=10, max_len=100, min_val=-100.0, max_val=100.0, dtype="float32"), - ndarrays(min_len=10, max_len=100, min_val=-100.0, max_val=100.0, dtype="float32"), + ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float32"), + ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float32"), ) def test_memoryview_float_noconj(A, B): if len(A) < len(B): diff --git a/tests/test_gemm.py b/tests/test_gemm.py index f244026b..67317322 100644 --- a/tests/test_gemm.py +++ b/tests/test_gemm.py @@ -48,8 +48,8 @@ def test_incompatible_shape(): @given( - ndarrays(min_len=10, max_len=100, min_val=-100.0, max_val=100.0, dtype="float64"), - ndarrays(min_len=10, max_len=100, min_val=-100.0, max_val=100.0, dtype="float64"), + ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float64"), + ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float64"), integers(min_value=2, max_value=1000), integers(min_value=2, max_value=1000), integers(min_value=2, max_value=1000), @@ -68,8 +68,8 @@ def test_memoryview_double_notrans(A, B, a_rows, a_cols, out_cols): @given( - ndarrays(min_len=10, max_len=100, min_val=-100.0, max_val=100.0, dtype="float32"), - ndarrays(min_len=10, max_len=100, min_val=-100.0, max_val=100.0, dtype="float32"), + ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float32"), + ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float32"), integers(min_value=2, max_value=1000), integers(min_value=2, max_value=1000), integers(min_value=2, max_value=1000), From 3d0f9ad1a680de8ad2e8d3e4b1cec77b3fc8aa9d Mon Sep 17 00:00:00 2001 From: crusaderky Date: Mon, 8 Dec 2025 17:51:12 +0000 Subject: [PATCH 3/6] Add central control for number of examples tested This is meant to be tampered with locally for more thorough (slower) tests. --- tests/blis_tests_common.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/blis_tests_common.py b/tests/blis_tests_common.py index 2f63bf80..5c984e9c 100644 --- a/tests/blis_tests_common.py +++ b/tests/blis_tests_common.py @@ -3,9 +3,12 @@ np.random.seed(0) -from hypothesis.strategies import tuples, integers, floats +from hypothesis import settings +from hypothesis.strategies import integers, floats from hypothesis.extra.numpy import arrays +# Increase this to run more thorough tests +hypothesis_default_profile = settings.register_profile("default", max_examples=100) def lengths(lo, hi): From 0fe793e469de457bedfa523388e40d386de36366 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Mon, 8 Dec 2025 17:38:09 +0000 Subject: [PATCH 4/6] New test for dotv invalid use case Mirrors same test in test_gemm.py. --- tests/test_dotv.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_dotv.py b/tests/test_dotv.py index ba530e0c..736bbbb8 100644 --- a/tests/test_dotv.py +++ b/tests/test_dotv.py @@ -1,4 +1,6 @@ # Copyright ExplosionAI GmbH, released under BSD. +import numpy as np +import pytest from hypothesis import given, assume from numpy.testing import assert_allclose @@ -6,6 +8,11 @@ from blis.py import dotv +def test_incompatible_shape(): + with pytest.raises(ValueError): + dotv(np.zeros(2), np.zeros(3)) + + @given( ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float64"), ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float64"), From 5c90b6938becbb97e8df7110e01ae0a746414bb5 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Mon, 8 Dec 2025 17:41:37 +0000 Subject: [PATCH 5/6] Overaul hypothesis strategies Improve readability. Given infinite hypothesis examples, this commit does not introduce any functional changes. However, given a fixed and relatively low max_examples setting, it substantially increases test coverage: 1. It prevents examples that were previously skipped by assume(). According to pytest --hypothesis-show-statistics, these were ~10% for dotv and ~20% for gemm. 2. It prevents examples that ended up being duplicates due to trimming. For example, in dotv, hypothesis could previously generate examples e.g. A=[1,2,3,4], B=[5,6] and A=[1,2], B=[5,6,7,8]; both would get trimmed to A=[1,2], B=[5,6]. 3. In gemm, it removes an entire degree of freedom by removing unused variable out_col, which again would result in duplicate examples. --- tests/blis_tests_common.py | 49 +++++++++------------- tests/test_dotv.py | 48 ++++++++------------- tests/test_gemm.py | 85 +++++++++----------------------------- 3 files changed, 57 insertions(+), 125 deletions(-) diff --git a/tests/blis_tests_common.py b/tests/blis_tests_common.py index 5c984e9c..456119fa 100644 --- a/tests/blis_tests_common.py +++ b/tests/blis_tests_common.py @@ -1,34 +1,25 @@ # Copyright ExplosionAI GmbH, released under BSD. -import numpy as np - -np.random.seed(0) - from hypothesis import settings -from hypothesis.strategies import integers, floats +from hypothesis import strategies as st from hypothesis.extra.numpy import arrays # Increase this to run more thorough tests -hypothesis_default_profile = settings.register_profile("default", max_examples=100) - - -def lengths(lo, hi): - return integers(min_value=lo, max_value=hi) - - -def ndarrays_of_shape(shape, lo, hi, dtype): - width = 64 if dtype == "float64" else 32 - return arrays( - dtype, - shape=shape, - elements=floats( - min_value=lo, - max_value=hi, - width=width, - ), - ) - - -def ndarrays(min_len, max_len, min_val, max_val, dtype): - return lengths(lo=min_len, hi=max_len).flatmap( - lambda n: ndarrays_of_shape(n, lo=min_val, hi=max_val, dtype=dtype) - ) +hypothesis_default_profile = settings.register_profile("default", max_examples=200) + +# (dtype, atol, rtol) +dtypes_tols = st.sampled_from( + [ + # FIXME huge tolerance needed for float32: + # https://github.com/explosion/cython-blis/issues/142 + ("float32", 1e-2, 1e-4), + ("float64", 1e-9, 1e-9), + ] +) + + +def ndarrays(shape, lo, hi, dtype): + """Draw ND NumPy arrays of floats""" + assert isinstance(dtype, str) and dtype.startswith("float") + width = int(dtype[5:]) + elements = st.floats(min_value=lo, max_value=hi, width=width) + return arrays(dtype, shape=shape, elements=elements) diff --git a/tests/test_dotv.py b/tests/test_dotv.py index 736bbbb8..a5425684 100644 --- a/tests/test_dotv.py +++ b/tests/test_dotv.py @@ -1,45 +1,31 @@ # Copyright ExplosionAI GmbH, released under BSD. import numpy as np import pytest -from hypothesis import given, assume - +from hypothesis import given +from hypothesis import strategies as st from numpy.testing import assert_allclose -from blis_tests_common import ndarrays + +from blis_tests_common import dtypes_tols, ndarrays from blis.py import dotv +@st.composite +def dotv_arrays(draw): + """Draw two 1D NumPy arrays of the same size and dtype""" + size = draw(st.integers(min_value=1, max_value=100)) + dtype, atol, rtol = draw(dtypes_tols) + arr_st = ndarrays((size,), lo=-100.0, hi=100.0, dtype=dtype) + return draw(arr_st), draw(arr_st), atol, rtol + + def test_incompatible_shape(): with pytest.raises(ValueError): dotv(np.zeros(2), np.zeros(3)) -@given( - ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float64"), - ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float64"), -) -def test_memoryview_double_noconj(A, B): - if len(A) < len(B): - B = B[: len(A)] - else: - A = A[: len(B)] - assume(A is not None) - assume(B is not None) - numpy_result = A.dot(B) - result = dotv(A, B) - assert_allclose(result, numpy_result, atol=1e-9, rtol=1e-9) - - -@given( - ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float32"), - ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float32"), -) -def test_memoryview_float_noconj(A, B): - if len(A) < len(B): - B = B[: len(A)] - else: - A = A[: len(B)] - assume(A is not None) - assume(B is not None) +@given(dotv_arrays()) +def test_memoryview_noconj(arrays): + A, B, atol, rtol = arrays numpy_result = A.dot(B) result = dotv(A, B) - assert_allclose(result, numpy_result, atol=1e-2, rtol=1e-4) + assert_allclose(result, numpy_result, atol=atol, rtol=rtol) diff --git a/tests/test_gemm.py b/tests/test_gemm.py index 67317322..15fa7fca 100644 --- a/tests/test_gemm.py +++ b/tests/test_gemm.py @@ -1,39 +1,25 @@ # Copyright ExplosionAI GmbH, released under BSD. -from math import sqrt, floor - -from hypothesis import given, assume -from hypothesis.strategies import integers - import numpy as np -from numpy.testing import assert_allclose import pytest +from hypothesis import given +from hypothesis import strategies as st +from numpy.testing import assert_allclose -from blis_tests_common import ndarrays +from blis_tests_common import dtypes_tols, ndarrays from blis.py import gemm -def _stretch_matrix(data, m, n): - ratio = sqrt(len(data) / (m * n)) - m = int(floor(m * ratio)) - n = int(floor(n * ratio)) - data = np.ascontiguousarray(data[: m * n], dtype=data.dtype) - return data.reshape((m, n)), m, n - - -def _reshape_for_gemm( - A, B, a_rows, a_cols, out_cols, dtype, trans_a=False, trans_b=False -): - A, a_rows, a_cols = _stretch_matrix(A, a_rows, a_cols) - if len(B) < a_cols or a_cols < 1: - return (None, None, None) - b_cols = int(floor(len(B) / a_cols)) - B = np.ascontiguousarray(B.flatten()[: a_cols * b_cols], dtype=dtype) - B = B.reshape((a_cols, b_cols)) - out_cols = B.shape[1] - C = np.zeros(shape=(A.shape[0], B.shape[1]), dtype=dtype) - if trans_a: - A = np.ascontiguousarray(A.T, dtype=dtype) - return A, B, C +@st.composite +def gemm_arrays(draw): + """Draw two NumPy arrays with shapes (l, m) and (m, n) of the same dtype""" + max_size = 100 + l = draw(st.integers(min_value=1, max_value=max_size)) # noqa: E741 + n = draw(st.integers(min_value=1, max_value=max_size)) + m = draw(st.integers(min_value=1, max_value=max_size // max(l, n))) + dtype, atol, rtol = draw(dtypes_tols) + A = draw(ndarrays((l, m), lo=-100.0, hi=100.0, dtype=dtype)) + B = draw(ndarrays((m, n), lo=-100.0, hi=100.0, dtype=dtype)) + return A, B, atol, rtol def test_incompatible_shape(): @@ -47,41 +33,10 @@ def test_incompatible_shape(): gemm(np.zeros((3, 2)), np.zeros((3, 2)), trans1=True, trans2=True) -@given( - ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float64"), - ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float64"), - integers(min_value=2, max_value=1000), - integers(min_value=2, max_value=1000), - integers(min_value=2, max_value=1000), -) -def test_memoryview_double_notrans(A, B, a_rows, a_cols, out_cols): - A, B, C = _reshape_for_gemm(A, B, a_rows, a_cols, out_cols, "float64") - assume(A is not None) - assume(B is not None) - assume(C is not None) - assume(A.size >= 1) - assume(B.size >= 1) - assume(C.size >= 1) - gemm(A, B, out=C) +@given(gemm_arrays()) +def test_memoryview_notrans(arrays): + A, B, atol, rtol = arrays numpy_result = A.dot(B) - assert_allclose(C, numpy_result, atol=1e-9, rtol=1e-9) - - -@given( - ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float32"), - ndarrays(min_len=1, max_len=100, min_val=-100.0, max_val=100.0, dtype="float32"), - integers(min_value=2, max_value=1000), - integers(min_value=2, max_value=1000), - integers(min_value=2, max_value=1000), -) -def test_memoryview_float_notrans(A, B, a_rows, a_cols, out_cols): - A, B, C = _reshape_for_gemm(A, B, a_rows, a_cols, out_cols, dtype="float32") - assume(A is not None) - assume(B is not None) - assume(C is not None) - assume(A.size >= 1) - assume(B.size >= 1) - assume(C.size >= 1) + C = np.zeros_like(numpy_result) # (l, n) gemm(A, B, out=C) - numpy_result = A.dot(B) - assert_allclose(C, numpy_result, atol=1e-2, rtol=1e-4) + assert_allclose(C, numpy_result, atol=atol, rtol=rtol) From 555d35f0c8568c2040bc5cd5f6dc3059ef771580 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Wed, 3 Dec 2025 16:04:12 +0000 Subject: [PATCH 6/6] Add threading tests for shared input --- pyproject.toml | 4 ++++ tests/blis_tests_common.py | 10 ++++++++++ tests/test_dotv.py | 16 +++++++++++++++- tests/test_gemm.py | 17 ++++++++++++++++- 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 81efdd4a..4b868ef2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,3 +63,7 @@ addopts = "--strict-markers --strict-config -v -r sxfE --color=yes --durations=1 python_files = ["test_*.py"] testpaths = ["tests"] filterwarnings = ["error"] +markers = [ + # Needed when pytest-run-parallel is not installed + "thread_unsafe: Run test in single thread in pytest-run-parallel" +] diff --git a/tests/blis_tests_common.py b/tests/blis_tests_common.py index 456119fa..7558c380 100644 --- a/tests/blis_tests_common.py +++ b/tests/blis_tests_common.py @@ -1,4 +1,5 @@ # Copyright ExplosionAI GmbH, released under BSD. +from concurrent.futures import ThreadPoolExecutor from hypothesis import settings from hypothesis import strategies as st from hypothesis.extra.numpy import arrays @@ -23,3 +24,12 @@ def ndarrays(shape, lo, hi, dtype): width = int(dtype[5:]) elements = st.floats(min_value=lo, max_value=hi, width=width) return arrays(dtype, shape=shape, elements=elements) + + +def run_threaded(func, max_workers=8, outer_iterations=2): + """Runs a function many times in parallel""" + with ThreadPoolExecutor(max_workers=max_workers) as tpe: + for _ in range(outer_iterations): + futures = [tpe.submit(func) for _ in range(max_workers)] + for f in futures: + f.result() diff --git a/tests/test_dotv.py b/tests/test_dotv.py index a5425684..8e621468 100644 --- a/tests/test_dotv.py +++ b/tests/test_dotv.py @@ -5,7 +5,7 @@ from hypothesis import strategies as st from numpy.testing import assert_allclose -from blis_tests_common import dtypes_tols, ndarrays +from blis_tests_common import dtypes_tols, ndarrays, run_threaded from blis.py import dotv @@ -29,3 +29,17 @@ def test_memoryview_noconj(arrays): numpy_result = A.dot(B) result = dotv(A, B) assert_allclose(result, numpy_result, atol=atol, rtol=rtol) + + +@given(dotv_arrays()) +@pytest.mark.thread_unsafe(reason="Uses run_threaded") +def test_threads_share_input(arrays): + """Test when multiple threads share the same input arrays.""" + A, B, atol, rtol = arrays + numpy_result = A.dot(B) + + def f(): + result = dotv(A, B) + assert_allclose(result, numpy_result, atol=atol, rtol=rtol) + + run_threaded(f) diff --git a/tests/test_gemm.py b/tests/test_gemm.py index 15fa7fca..7ef90eb2 100644 --- a/tests/test_gemm.py +++ b/tests/test_gemm.py @@ -5,7 +5,7 @@ from hypothesis import strategies as st from numpy.testing import assert_allclose -from blis_tests_common import dtypes_tols, ndarrays +from blis_tests_common import dtypes_tols, ndarrays, run_threaded from blis.py import gemm @@ -40,3 +40,18 @@ def test_memoryview_notrans(arrays): C = np.zeros_like(numpy_result) # (l, n) gemm(A, B, out=C) assert_allclose(C, numpy_result, atol=atol, rtol=rtol) + + +@given(gemm_arrays()) +@pytest.mark.thread_unsafe(reason="Uses run_threaded") +def test_threads_share_input(arrays): + """Test when multiple threads share the same input arrays.""" + A, B, atol, rtol = arrays + numpy_result = A.dot(B) + + def f(): + C = np.zeros_like(numpy_result) # (l, n) + gemm(A, B, out=C) + assert_allclose(C, numpy_result, atol=atol, rtol=rtol) + + run_threaded(f)