Merge pull request #9 from hpyproject/hpy

Update the HPy branch

Author: paugier

.github/workflows/tests.yml

@@ -0,0 +1,50 @@
+name: Tests
+
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 5
+      matrix:
+        python-version: ['3.8', '3.9', '3.10']
+
+    steps:
+
+    - name: Setup Julia
+      uses: julia-actions/setup-julia@v1
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        # install HPy from source if depending on a dev version
+        # git clone -b master --single-branch https://github.com/hpyproject/hpy
+        # git checkout 1234abcd
+        # cd hpy
+        # pip install .
+        pip install numpy cython pytest transonic pythran 'setuptools>=60.2' 'hpy>=0.9.0rc1'
+
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: build
+      run: |
+        python setup.py develop
+        python setup.py --hpy-abi=universal develop
+
+    - name: Run tests
+      run: |
+        pytest -s
+
+    - name: Run bench
+      run: |
+        cd bench
+        make bench_hpy
+        make

.gitignore

@@ -2,6 +2,7 @@
 *.egg-info/*
 */__pycache__/*
 build
+.eggs
 
 *.so
 .vscode

.travis.yml

@@ -1,6 +1,7 @@
 BSD 3-Clause License
 
 Copyright (c) 2020, Pierre Augier
+Copyright (c) 2021, 2023, Oracle and/or it's affiliates
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

LICENSE

@@ -4,8 +4,27 @@ PYTHON := python
 endif
 
 all:
+	make develop_universal
+ifeq ($(PYTHON),python)
+	make build_ext
+endif
+
+develop:
+	$(PYTHON) setup.py develop
+
+develop_universal:
+	$(PYTHON) setup.py --hpy-abi=universal develop
+	rm -f piconumpy/_piconumpy_hpy.py
+
+pip:
 	$(PYTHON) -m pip install -e .[dev]
 
+build_ext_universal:
+	$(PYTHON) setup.py --hpy-abi=universal build_ext -if
+
+build_ext:
+	$(PYTHON) setup.py build_ext -if
+
 full:
 	$(PYTHON) -m pip install -e .[full]
 
@@ -18,6 +37,7 @@ tests:
 
 clean:
 	rm -f piconumpy/*.so
+	rm -rf build dist piconumpy.egg-info
 
 black:
 	black -l 82 .

Makefile

@@ -2,7 +2,7 @@
 
 [![Build Status](https://travis-ci.org/paugier/piconumpy.svg?branch=master)](https://travis-ci.org/paugier/piconumpy)
 
-**An experiment about Numpy and pyhandle/hpy**
+**An experiment about Numpy and HPy**
 
 The C API of CPython is one of the cause of the success of Python in scientific
 computing. In particular, Numpy (and all the Python scientific stack) is built
@@ -11,7 +11,7 @@ issue for the future of scientific Python (see [1], [2], [HPy]).
 
 [1]: https://faster-cpython.readthedocs.io/
 [2]: https://morepypy.blogspot.com/2019/12/hpy-kick-off-sprint-report.html
-[HPy]: https://github.com/pyhandle/hpy
+[HPy]: https://github.com/hpyproject/hpy
 
 [HPy] is a very ambitious and promissing project to design a new and better C
 API for interacting with Python interpreters. It should allow people to write
@@ -60,6 +60,11 @@ such codes using Numpy).
 
 ## Install and run the benchmarks
 
+**Warning:** PicoNumpy now depends on HPy, which still has to be installed from
+the [Git repository](https://github.com/hpyproject/hpy). For now, the
+installation is a bit more complex that what is described here (more about this
+[here](#more-precise-notes-on-how-to-install-and-run-the-benchmarks-with-PyPy)).
+
 `make` should install the package in editable mode. `cd bench; make` should run
 the benchmarks. For the benchmarks, Julia is used for a good comparison point
 so the command `julia` has to be available.
@@ -85,106 +90,97 @@ make profile METHOD="purepy"
 make profile METHOD="cython"
 ```
 
+### More precise notes on how to install and run the benchmarks with PyPy
+
+Download and extract a nightly PyPy build
+<https://buildbot.pypy.org/nightly/>. Add to the `PATH` environment variable
+the path of the directory containing the `pypy` executable (something like
+`~/opt/pypy-c-jit-101190-b661dc329618-linux64/bin`). Then, you should be able
+to run:
+
+```bash
+pypy -m ensurepip
+pypy -m pip install pip -U
+pypy -m pip install numpy cython pytest transonic pythran
+```
+
+We need to install the correct version of HPy for the version of PyPy we are using:
+
+```bash
+pypy -c "import hpy.universal as u; print(u.get_version())"
+```
+
+gives `('0.0.2rc2.dev12+gc9660c2', 'c9660c2')`.
+
+```bash
+cd ~/Dev/hpy
+# update to the correct commit
+pypy setup.py develop
+```
+
+Now we can build-install PicoNumpy:
+
+```bash
+cd ~/Dev/piconumpy
+pypy setup.py --hpy-abi=universal develop
+```
+
+And run the benchmarks with:
+
+```bash
+export PYTHON="pypy"
+make clean
+make bench_hpy
+make
+```
+
 ## Few results
 
-As of today (5 March 2020), HPy is not yet ready!
+As of today (6 July 2021), HPy is not yet ready for high performance, but at
+least (with HPy 0.0.2) it runs !
 
-### At work (meige8pcpa79, Intel(R) Xeon(R) CPU E5-1603 v3 @ 2.80GHz)
+### At home (Intel(R) Core(TM) i5-8400 CPU @ 2.80GHz)
 
 - With CPython
 
-```raw
-Julia                      :     1 * norm = 0.00534 s
-Transonic-Pythran          : 0.564 * norm
-Numpy                      :  18.2 * norm
-PicoNumpy (purepy)         :  23.7 * norm
-PicoNumpy (purepy_array)   :  21.7 * norm
-PicoNumpy (Cython)         :  15.6 * norm
-PicoNumpy (CPython C-API)  :   4.7 * norm
+```
+Julia                      :     1 * norm = 0.00196 s
+PicoNumpy (CPython C-API)  :  9.42 * norm
+PicoNumpy (HPy CPy ABI)    :  9.95 * norm
+PicoNumpy (HPy Universal)  :  10.4 * norm
+Transonic-Pythran          : 0.497 * norm
+Numpy                      :  27.5 * norm
+PicoNumpy (purepy)         :  37.3 * norm
+PicoNumpy (purepy_array)   :  37.7 * norm
+PicoNumpy (Cython)         :  28.9 * norm
 ```
 
 - With PyPy3
 
-```raw
-Julia                      :     1 * norm = 0.00534 s
-Transonic-Pythran          : 0.603 * norm
-Numpy                      :   183 * norm
-PicoNumpy (purepy)         :   2.4 * norm
-PicoNumpy (purepy_array)   :  3.73 * norm
-PicoNumpy (Cython)         :   135 * norm
-PicoNumpy (CPython C-API)  :  17.8 * norm
+```
+Julia                      :     1 * norm = 0.00196 s
+PicoNumpy (CPython C-API)  :  34.1 * norm
+PicoNumpy (HPy Universal)  :  12.8 * norm
+Transonic-Pythran          : 0.539 * norm
+Numpy                      :   232 * norm
+PicoNumpy (purepy)         :  4.39 * norm
+PicoNumpy (purepy_array)   :  6.33 * norm
+PicoNumpy (Cython)         :   274 * norm
 ```
 
-### At home (Intel(R) Core(TM) i5-8400 CPU @ 2.80GHz)
+#### Simpler benchmarks (bench/bench_cpy_vs_hpy.py)
 
 - With CPython
 
-```raw
-Julia                      :     1 * norm = 0.00176 s
-Transonic-Pythran          :  0.73 * norm
-Numpy                      :  37.6 * norm
-PicoNumpy (purepy)         :  46.4 * norm
-PicoNumpy (purepy_array)   :  41.9 * norm
-PicoNumpy (Cython)         :  31.4 * norm
-PicoNumpy (CPython C-API)  :  9.22 * norm
+```
+CPython C-API:   1.92 seconds
+HPy [Universal]: 2.08 seconds
+HPy [CPy ABI]:   2.02 seconds
 ```
 
 - With PyPy3
 
-```raw
-Julia                      :     1 * norm = 0.00176 s
-Transonic-Pythran          : 0.797 * norm
-Numpy                      :   326 * norm
-PicoNumpy (purepy)         :  4.71 * norm
-PicoNumpy (purepy_array)   :  7.48 * norm
-PicoNumpy (Cython)         :   243 * norm
-PicoNumpy (CPython C-API)  :  36.6 * norm
 ```
-
-## CPython C-API usage in PicoNumpy
-
-It is the first time that I wrote a Python extension by hand (without Cython or
-Pythran) and moreover, there are nearly no checks and I didn't pay attention to
-reference counting!
-
-List obtained from `grep -rwoh 'Py[_A-Z].[a-zA-Z_]*' | sort | uniq`:
-
-```raw
-PyArg_ParseTupleAndKeywords
-PyCFunction
-Py_DECREF
-PyErr_NoMemory
-PyErr_SetString
-PyExc_TypeError
-PyFloat_AsDouble
-PyFloat_FromDouble
-PyFloatObject
-Py_INCREF
-PyList_Check
-PyList_GET_ITEM
-PyList_New
-PyList_SetItem
-PyList_Size
-PyLong_AsLong
-PyMemberDef
-PyMethodDef
-PyMODINIT_FUNC
-PyModule_AddObject
-PyModule_Create
-PyModuleDef
-PyModuleDef_HEAD_INIT
-PyNumber_Check
-PyNumberMethods
-PyObject
-PyObject_HEAD
-PyObject_New
-PySequenceMethods
-Py_ssize_t
-Py_TPFLAGS_DEFAULT
-Py_TYPE
-PyType_GenericNew
-PyTypeObject
-PyType_Ready
-Py_UNUSED
-PyVarObject_HEAD_INIT
+CPython C-API:   5.75 seconds
+HPy [Universal]: 2.11 seconds
 ```

README.md

@@ -24,4 +24,7 @@ profile: tmp.py
 	$(PYTHON) profile_piconumpy.py $(METHOD)
 	# with gprof2dot and graphviz (command dot)
 	gprof2dot -f pstats tmp.pstats | dot -Tpng -o tmp_$(METHOD).png
-	eog tmp_$(METHOD).png
+	eog tmp_$(METHOD).png
+
+bench_hpy:
+	$(PYTHON) bench_cpy_vs_hpy.py

bench/Makefile

@@ -13,7 +13,6 @@ function board(X_0::Array, b::Float64)
     p = (2*π)/10.0
     q = (2*π)/4.0
 
-    H = -a*x0 - b*cos(p*x0)*cos(q*y0)
     H_x = -a + b*p*sin(p*x0)*cos(q*y0)
     H_xx = b*p^2 * cos(p*x0)*cos(q*y0)
     H_y = b*q*cos(p*x0)*sin(q*y0)

bench/bench.jl

@@ -13,7 +13,10 @@ def runge_kutta_step(f, x0, dt, t=None):
     k2 = f(t, x0 + k1 / 2) * dt
     k3 = f(t, x0 + k2 / 2) * dt
     k4 = f(t, x0 + k3) * dt
-    x_new = x0 + (k1 + 2 * k2 + 2 * k3 + k4) / 6
+    # workaround for a pypy bug
+    # see https://foss.heptapod.net/pypy/pypy/-/issues/3509
+    # x_new = x0 + (k1 + 2 * k2 + 2 * k3 + k4) / 6
+    x_new = x0 + (k1 + k2 * 2 + k3 * 2 + k4) / 6
     return x_new