diff --git a/CHANGES/1178.feature.rst b/CHANGES/1178.feature.rst new file mode 100644 index 000000000..36d981673 --- /dev/null +++ b/CHANGES/1178.feature.rst @@ -0,0 +1 @@ +Started exposing an interface for importing the :mod:`multidict` C-extension from downstream Cython libraries -- by :user:`Vizonex`. diff --git a/MANIFEST.in b/MANIFEST.in index 43625782e..12dfadfd2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,6 +12,7 @@ graft requirements graft tests global-exclude *.pyc include multidict/*.c +include multidict/_multilib/*.h exclude multidict/_multidict.html exclude multidict/*.so exclude multidict/*.pyd diff --git a/docs/in-cython.rst b/docs/in-cython.rst new file mode 100644 index 000000000..4e1ae6dca --- /dev/null +++ b/docs/in-cython.rst @@ -0,0 +1,101 @@ +.. _cython-api: + +========== +Cython API +========== + +Multidicts implements a cython api that is used for speeding up the aiohttp http parser +but this feature can be used elsewhere in your own projects. + + +Introduction +------------ +Multidict can be used with cython to speedup performance of other tools or +scripts you may think about programming. Those who are familliar with the way +`numpy `_ +works should know that this library works the exact same way. If your not familliar with this don't worry. + +An example might be combining the node-js `llhttp `_ library +and Multidict together for example (which is something aiohttp already +does) . By using llhttp's callback functions on items such as HTTP headers and URL +query arguments you can build some extremely fast parsers and more +with extra performance benefits included. + + +Functions for using MultiDict in cython +should have very simillar feel and format to the way CPython was written. e.g.: + +.. code-block:: python + + from multidict cimport import_multidict, MultiDict, MultiDict_Add + # always remeber to call import_multidict before anything else + # otherwise your compilation will fail + import_multidict() + + cdef MultiDict create_with_user_agent_header(): + cdef MultiDict md = MultiDict() + MultiDict_Add(md, "user-agent", "Multidict-Made-User-Agent") + return md + + + + + + +Compiling +--------- +Compiling multidict with cython works the exact same way as *numpy* with the only +requirement being to link where the headers needed to compile the library are kept +luckily multidict includes a function to get where the headers are stored called +*get_include* and it is no different from the way `numpy works `_. +e.g.: + +.. code-block:: python + + from Cython.Build import cythonize + from setuptools import Extension, setup + import multidict + + if __name__ == "__main__": + setup( + ext_modules=cythonize( + Extension( + "your_module.pyx", sources=["your_module.pyx"] + ) + ), + # in here is where you could but down your include directories + include_dirs=[multidict.get_include()], + ) + + +Know that your are not limited to just one *include_dirs* directory in fact you could combine +*numpy* and *multidict* together if you really wanted to along with other C Libraries that you would +like to compile alongside it. e.g.: + +.. code-block:: python + + from setuptools import Extension, setup + from Cython.Build import cythonize + import numpy + import multidict + + extensions = [ + Extension("*", ["*.pyx"], + include_dirs=[ + numpy.get_include(), + multidict.get_include(), + "my-other-clibraries/path/etc" + ] + ), + ] + setup( + name="My hello app", + ext_modules=cythonize(extensions), + ) + +There's + + + + + diff --git a/docs/index.rst b/docs/index.rst index 806a46c12..b7fefe918 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -101,6 +101,7 @@ Contents multidict benchmark changes + in-cython Indices and tables ================== diff --git a/docs/multidict.rst b/docs/multidict.rst index 33c72993a..42f09eca3 100644 --- a/docs/multidict.rst +++ b/docs/multidict.rst @@ -427,6 +427,253 @@ The type of multidict keys is always :class:`str` or a class derived from a stri .. versionadded:: 3.7 +Cython API Reference +==================== + +This library is also shipped with a Cython API + +.. function:: int MultiDict_GetAll(MultiDict self, object key, PyObject **ret) except -1 + + functions the same way as the :class:`MultiDict` *getall* function but it is binded to the C function directly under the + hood. Returns -1 if the key does not exist, and the return value will be *NULL*. Know that this doesn't raise a *KeyError* + And it is excepted the that the end programmer would take care of that. + + e.g:: + + from cpython.object cimport PyObject + + def get_all_of(MultiDict md): + cdef PyObject* ret + # This will raise KeyError if we didn't handle the exception in a different way. + if MultiDict_GetAll(md, "key", &ret) < 0: + raise KeyError("key") + + +.. function:: int MultiDict_GetOne(MultiDict self, object key, PyObject **ret) except -1 + + functions the same way as :class:`MultiDict` *getone* function but is binded directly to C. + Returns -1 if Key was not found. This function does not raise *KeyError* and it's excepted + for the end developer to take care of raising that exception. + +.. function:: object MultiDict_Keys(MultiDict self) + + function works the same as :class:`MultiDict` *keys* function. + + e.g:: + + cdef iterate_md(MultiDict md): + cdef object key + for key in MultiDict_Keys(md): + ... + + +.. function:: object MultiDict_Items(MultiDict self) + + function works the same as :class:`MultiDict` *items* function. + + e.g:: + + cdef items_md(MultiDict md): + cdef object key, value + for key, value in MultiDict_Items(md): + ... + + + .. function:: object MultiDict_Values(MultiDict self) + + function works the same as :class:`MultiDict` *values* function. + + e.g:: + + cdef iterate_md(MultiDict md): + cdef object value + for value in MultiDict_Values(md): + ... + + .. function:: int MultiDict_Add(MultiDict self, object key, object value) except -1 + + function adds a key and value to a :class:`MultiDict` returns -1 on failure + + e.g:: + + if MultiDict_Add(md, "foo", "spam") < 0: + raise RuntimeError("Failed to add (foo, spam)") + + + .. function:: PyObject* MultiDict_Extend(MultiDict self, tuple args, dict kwargs) except NULL + + function extends a multidict with both args or kwargs. Ignoring one or the other should work. + returns a *NULL* pointer if all else fails. + + e.g:: + + if MultiDict_Extend(md, [("a", 1)], {"k": 1}) == NULL: + ... + + + .. function:: MultiDict MultiDict_Copy(MultiDict self) + + copies a multidict and returns a new one with the copied items. + + .. function:: PyObject* MultiDict_SetDefault(MultiDict self, object key, object value) except NULL + + works the same as :class:`MultiDict` *setdefault*, returns `NULL` if something fails. + + .. function:: int MultiDict_PopOne(MultiDict self, object key, PyObject** ret) except -1 + + works the same as :class:`MultiDict` *popone*, returns -1 if something fails. + + .. function:: int MultiDict_PopAll(MultiDict self, object key, PyObject** ret) except -1 + + works the same as :class:`MultiDict` *popall*, returns -1 if something fails. + + .. function:: object MultiDict_PopItem(MultiDict self) + + works the same as :class:`MultiDict` *popitem*, raises :exc:`KeyError` if dictionary is empty + + .. function:: PyObject* MultiDict_Update(MultiDict self, tuple args, dict kwds) except NULL + + works the same as :class:`MultiDict` *update*, returns NULL if the function fails to update something + + e.g:: + + MultiDict_Update(md, (), {"key":"value"}) + + +.. function:: int CIMultiDict_GetAll(CIMultiDict self, object key, PyObject **ret) except -1 + + functions the same way as the :class:`CIMultiDict` *getall* function but it is binded to the C function directly under the + hood. Returns -1 if the key does not exist, and the return value will be *NULL*. Know that this doesn't raise a *KeyError* + And it is excepted the that the end programmer would take care of that. + + e.g:: + + from cpython.object cimport PyObject + + def get_all_of(CIMultiDict md): + cdef PyObject* ret + # This will raise KeyError if we didn't handle the exception in a different way. + if CIMultiDict_GetAll(md, "key", &ret) < 0: + raise KeyError("key") + + +.. function:: int CIMultiDict_GetOne(CIMultiDict self, object key, PyObject **ret) except -1 + + functions the same way as :class:`CIMultiDict` *getone* function but is binded directly to C. + Returns -1 if Key was not found. This function does not raise *KeyError* and it's excepted + for the end developer to take care of raising that exception. + +.. function:: object CIMultiDict_Keys(CIMultiDict self) + + function works the same as :class:`CIMultiDict` *keys* function. + + e.g:: + + cdef iterate_md(CIMultiDict md): + cdef object key + for key in CIMultiDict_Keys(md): + ... + + +.. function:: object CIMultiDict_Items(CIMultiDict self) + + function works the same as :class:`CIMultiDict` *items* function. + + e.g:: + + cdef items_md(CIMultiDict md): + cdef object key, value + for key, value in CIMultiDict_Items(md): + ... + + + .. function:: object CIMultiDict_Values(CIMultiDict self) + + function works the same as :class:`CIMultiDict` *values* function. + + e.g:: + + cdef iterate_md(CIMultiDict md): + cdef object value + for value in CIMultiDict_Values(md): + ... + + .. function:: int CIMultiDict_Add(CIMultiDict self, object key, object value) except -1 + + function adds a key and value to a :class:`CIMultiDict` returns -1 on failure + + e.g:: + + if CIMultiDict_Add(md, "foo", "spam") < 0: + raise RuntimeError("Failed to add (foo, spam)") + + + .. function:: PyObject* CIMultiDict_Extend(CIMultiDict self, tuple args, dict kwargs) except NULL + + function extends a CImultidict with both args or kwargs. Ignoring one or the other should work. + returns a *NULL* pointer if all else fails. + + e.g:: + + if CIMultiDict_Extend(md, [("a", 1)], {"k": 1}) == NULL: + ... + + + .. function:: CIMultiDict CIMultiDict_Copy(CIMultiDict self) + + copies a CImultidict and returns a new one with the copied items. + + .. function:: PyObject* CIMultiDict_SetDefault(CIMultiDict self, object key, object value) except NULL + + works the same as :class:`CIMultiDict` *setdefault*, returns `NULL` if something fails. + + .. function:: int CIMultiDict_PopOne(CIMultiDict self, object key, PyObject** ret) except -1 + + works the same as :class:`CIMultiDict` *popone*, returns -1 if something fails. + + .. function:: int CIMultiDict_PopAll(CIMultiDict self, object key, PyObject** ret) except -1 + + works the same as :class:`CIMultiDict` *popall*, returns -1 if something fails. + + .. function:: object CIMultiDict_PopItem(CIMultiDict self) + + works the same as :class:`CIMultiDict` *popitem*, raises :exc:`KeyError` if dictionary is empty + + .. function:: PyObject* CIMultiDict_Update(CIMultiDict self, tuple args, dict kwds) except NULL + + works the same as :class:`CIMultiDict` *update*, returns NULL if the function fails to update something + + e.g:: + + CIMultiDict_Update(md, (), {"key":"value"}) + + .. function:: int MultiDictProxy_GetAll(MultiDictProxy self, object key, PyObject **ret) + + todo... + + .. function:: int MultiDictProxy_GetOne(MultiDictProxy self, object key, PyObject **ret) + + todo... + + .. function:: object MultiDictProxy_Keys(MultiDictProxy self) + + todo... + + .. function:: object MultiDictProxy_Values(MultiDictProxy self) + + todo... + + .. funciton:: object MultiDictProxy_Items(MultiDictProxy self) + + todo... + + .. funciton:: MultiDictProxy MultiDictProxy_Copy(MultiDictProxy self) + + todo... + + + + Environment variables ===================== diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index d422de0f2..e3e04a97c 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -68,6 +68,7 @@ multipart Multipart mypy Nikolay +numpy param params performant diff --git a/multidict/__init__.pxd b/multidict/__init__.pxd new file mode 100644 index 000000000..9711d293c --- /dev/null +++ b/multidict/__init__.pxd @@ -0,0 +1,130 @@ +# cython: language_level = 3, freethreading_compatible = True + +from cpython.object cimport PyObject + + +cdef extern from "Python.h": + void Py_INCREF(PyObject* o) + bint Py_IS_TYPE(object, type) + bint PyObject_TypeCheck(object, type) + + +cdef extern from "_multilib/dict.h": + + ctypedef struct MultiDictObject: + pass + + ctypedef struct MultiDictProxyObject: + pass + + ctypedef class multidict.MultiDict [object MultiDictObject, check_size ignore]: + pass + + ctypedef class multidict.MultiDictProxy [object MultiDictProxyObject, check_size ignore]: + cdef MultiDict md + + ctypedef class multidict.CIMultiDict [object MultiDictObject, check_size ignore]: + pass + + ctypedef class multidict.CIMultiDictProxy [object MultiDictProxyObject, check_size ignore]: + pass + + +cdef extern from "_multilib/istr.h": + """ +/* From multidict.__init__.pxd for _multilib/istr.h */ + +/* To ensure IStr_CheckExact works as if it were a CPython function + * A Simple Hack was required to bypass this issue */ + + + """ + + ctypedef struct istrobject: + pass + + ctypedef class multidict.istr [object istrobject, check_size ignore]: + cdef object canonical + pass + + + + +cdef extern from "_multilib/capsule.h": + + # ==================== MultiDict Functions ==================== + + int MultiDict_GetAll(MultiDict self, object key, PyObject **ret) except -1 + int MultiDict_GetOne(MultiDict self, object key, PyObject **ret) except -1 + object MultiDict_Keys(MultiDict self) + object MultiDict_Items(MultiDict self) + object MultiDict_Values(MultiDict self) + int MultiDict_Add(MultiDict self, object key, object value) + + PyObject* MultiDict_Clear(MultiDict self) except NULL + PyObject* MultiDict_Extend(MultiDict self, tuple args, dict kwargs) except NULL + MultiDict MultiDict_Copy(MultiDict self) + PyObject* MultiDict_SetDefault(MultiDict self, object key, object value) except NULL + + int MultiDict_PopOne(MultiDict self, object key, PyObject** ret) except -1 + int MultiDict_PopAll(MultiDict self, object key, PyObject** ret) except -1 + object MultiDict_PopItem(MultiDict self) + PyObject* MultiDict_Update(MultiDict self, tuple args, dict kwds) except NULL + + # ==================== CIMultiDict Functions ==================== + + int CIMultiDict_GetAll "MultiDict_GetAll" (CIMultiDict self, object key, PyObject **ret) + int CIMultiDict_GetOne "MultiDict_GetOne" (CIMultiDict self, object key, PyObject **ret) + object CIMultiDict_Keys "MultiDict_Keys" (CIMultiDict self) + object CIMultiDict_Items "MultiDict_Items" (CIMultiDict self) + object CIMultiDict_Values "MultiDict_Values" (CIMultiDict self) + int CIMultiDict_Add "MultiDict_Add" (CIMultiDict self, object key, object value) + + PyObject* CIMultiDict_Clear "MultiDict_Clear" (CIMultiDict self) except NULL + PyObject* CIMultiDict_Extend "MultiDict_Extend" (CIMultiDict self, tuple args, dict kwargs) except NULL + MultiDict CIMultiDict_Copy "MultiDict_Copy" (CIMultiDict self) + PyObject* CIMultiDict_SetDefault "MultiDict_SetDefault" (CIMultiDict self, object key, object value) except NULL + + int CIMultiDict_PopOne "MultiDict_PopOne" (CIMultiDict self, object key, PyObject** ret) + int CIMultiDict_PopAll "MultiDict_PopAll" (CIMultiDict self, object key, PyObject** ret) + object CIMultiDict_PopItem "MultiDict_PopItem" (CIMultiDict self) + PyObject* CIMultiDict_Update "MultiDict_Update" (CIMultiDict self, tuple args, dict kwds) except NULL + + # ==================== MultiDictProxy Functions ==================== + + int MultiDictProxy_GetAll(MultiDictProxy self, object key, PyObject **ret) except -1 + int MultiDictProxy_GetOne(MultiDictProxy self, object key, PyObject **ret) except -1 + object MultiDictProxy_Keys(MultiDictProxy self) + object MultiDictProxy_Values(MultiDictProxy self) + object MultiDictProxy_Items(MultiDictProxy self) + MultiDictProxy MultiDictProxy_Copy(MultiDictProxy self) + + # ==================== CIMultiDictProxy Functions ==================== + + int CIMultiDictProxy_GetAll "MultiDictProxy_GetAll" (MultiDictProxy self, object key, PyObject **ret) except -1 + int CIMultiDictProxy_GetOne "MultiDictProxy_GetOne"(MultiDictProxy self, object key, PyObject **ret) except -1 + object CIMultiDictProxy_Keys "MultiDictProxy_Keys" (MultiDictProxy self) + object CIMultiDictProxy_Values "MultiDictProxy_Values" (MultiDictProxy self) + object CIMutliDictProxy_Items "MultiDictProxy_Items" (MultiDictProxy self) + CIMultiDictProxy CIMultiDictProxy_Copy "MultiDictProxy_Copy"(CIMultiDictProxy self) + + # NOTE: Make sure you import this before using anything + int MultiDict_IMPORT() except -1 + int import_multidict "MultiDict_IMPORT" () except -1 + + +cdef inline object MultiDict_Get(MultiDict self, object key, object default = None): + cdef PyObject* ret + if MultiDict_GetOne(self, key, &ret) < 0: + return default + Py_INCREF(ret) + return ret + +# There is not currently good api to use for istr, +# so we just have to recreate what was in istr.h +cdef inline bint IStr_CheckExact (object obj): + return Py_IS_TYPE(obj, istr) + +cdef inline bint IStr_Check (object obj): + return IStr_CheckExact(obj) or PyObject_TypeCheck(obj, istr) + diff --git a/multidict/__init__.py b/multidict/__init__.py index 51a9ccc33..35bda8c8a 100644 --- a/multidict/__init__.py +++ b/multidict/__init__.py @@ -5,6 +5,7 @@ several values for the same key. """ +import pathlib from typing import TYPE_CHECKING from ._abc import MultiMapping, MutableMultiMapping @@ -20,6 +21,7 @@ "upstr", "istr", "getversion", + "get_include", ) __version__ = "6.5.1.dev0" @@ -57,3 +59,15 @@ upstr = istr + +# Inspired by Numpy + + +def get_include() -> str: + """ + Get multidict headers for compiling + multidict with other C Extensions or + cython code + """ + + return str(pathlib.Path(__file__).parent) diff --git a/multidict/_multidict.c b/multidict/_multidict.c index 7171cccc4..a48ee7608 100644 --- a/multidict/_multidict.c +++ b/multidict/_multidict.c @@ -1,6 +1,7 @@ #include #include +#include "_multilib/capsule.h" #include "_multilib/dict.h" #include "_multilib/hashtable.h" #include "_multilib/istr.h" @@ -237,7 +238,7 @@ multidict_copy(MultiDictObject *self) } static inline PyObject * -_multidict_proxy_copy(MultiDictProxyObject *self, PyTypeObject *type) +_multidict_proxy_copy(MultiDictProxyObject *self) { return multidict_copy(self->md); } @@ -1096,7 +1097,7 @@ multidict_proxy_values(MultiDictProxyObject *self) static PyObject * multidict_proxy_copy(MultiDictProxyObject *self) { - return _multidict_proxy_copy(self, self->md->state->MultiDictType); + return _multidict_proxy_copy(self); } static PyObject * @@ -1311,7 +1312,7 @@ cimultidict_proxy_tp_init(MultiDictProxyObject *self, PyObject *args, static PyObject * cimultidict_proxy_copy(MultiDictProxyObject *self) { - return _multidict_proxy_copy(self, self->md->state->CIMultiDictType); + return _multidict_proxy_copy(self); } PyDoc_STRVAR(CIMultDictProxy_doc, "Read-only proxy for CIMultiDict instance."); @@ -1362,6 +1363,26 @@ getversion(PyObject *self, PyObject *arg) return PyLong_FromUnsignedLong(md_version(md)); } +/********************* CAPI **********************/ + +// Not to be confused with the variable in capsule.h +// hence the underscores +static MultiDict_CAPI __MultiDict_API = { + ._MultiDict_GetAll = md_get_all, + ._MultiDict_GetOne = md_get_one, + ._MultiDict_Keys = multidict_keys, + ._MultiDict_Items = multidict_items, + ._MultiDict_Values = multidict_values, + ._MultiDict_Add = md_add, + ._MultiDict_Clear = multidict_clear, + ._MultiDict_Extend = multidict_extend, + ._MultiDict_Copy = multidict_copy, + ._MultiDict_SetDefault = md_set_default, + ._MultiDict_PopOne = md_pop_one, + ._MultiDict_PopItem = md_pop_item, + ._MultiDict_Update = multidict_update, +}; + /******************** Module ********************/ static int @@ -1520,6 +1541,21 @@ module_exec(PyObject *mod) goto fail; } + // Capsules can be confusing but turns out you have to give it + // the fullname, as long as it you do that it will import. + + PyObject *py_capi_obj = + PyCapsule_New((void *)(&__MultiDict_API), + "multidict._multidict.multidict_CAPI", + NULL); + if (py_capi_obj == NULL) { + goto fail; + } + + if (PyModule_Add(mod, "multidict_CAPI", py_capi_obj) < 0) { + goto fail; + }; + return 0; fail: Py_CLEAR(tpl); diff --git a/multidict/_multilib/capsule.h b/multidict/_multilib/capsule.h new file mode 100644 index 000000000..b24a6f527 --- /dev/null +++ b/multidict/_multilib/capsule.h @@ -0,0 +1,133 @@ +#ifndef __CAPSULE_H__ +#define __CAPSULE_H__ + +#include "Python.h" +#include "dict.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct _multidict_capi { + // md_get_all + int (*_MultiDict_GetAll)(MultiDictObject *self, PyObject *key, + PyObject **ret); + // md_get_one + int (*_MultiDict_GetOne)(MultiDictObject *self, PyObject *key, + PyObject **ret); + + // multidict_keys + PyObject *(*_MultiDict_Keys)(MultiDictObject *self); + + // multidict_items + PyObject *(*_MultiDict_Items)(MultiDictObject *self); + + // multidict_values + PyObject *(*_MultiDict_Values)(MultiDictObject *self); + + // md_add + int (*_MultiDict_Add)(MultiDictObject *self, PyObject *key, + PyObject *value); + + // multidict_clear + PyObject *(*_MultiDict_Clear)(MultiDictObject *self); + + // multidict_extend + PyObject *(*_MultiDict_Extend)(MultiDictObject *self, PyObject *args, + PyObject *kwargs); + + // multidict_copy + PyObject *(*_MultiDict_Copy)(MultiDictObject *self); + + // multidict_setdefault + PyObject *(*_MultiDict_SetDefault)(MultiDictObject *self, PyObject *key, + PyObject *value); + + // md_pop_one + int (*_MultiDict_PopOne)(MultiDictObject *self, PyObject *key, + PyObject **ret); + + // md_pop_all + int (*_MultiDict_PopAll)(MultiDictObject *self, PyObject *key, + PyObject **ret); + + // md_pop_item + PyObject *(*_MultiDict_PopItem)(MultiDictObject *self); + + // multidict_update + PyObject *(*_MultiDict_Update)(MultiDictObject *self, PyObject *args, + PyObject *kwds); + + // TODO: + // PyObject *(*_MultiDict_Delete)(MultiDictObject *self, PyObject *); + +} MultiDict_CAPI; + +static MultiDict_CAPI *MultiDictAPI = NULL; + +static int +MultiDict_IMPORT() +{ + MultiDictAPI = PyCapsule_Import("multidict._multidict.multidict_CAPI", 0); + return (MultiDictAPI != NULL) ? 0 : -1; +} + +/* MultiDict / CIMultiDict Macros */ + +#define MultiDict_GetAll(self, key, ret) \ + MultiDictAPI->_MultiDict_GetAll(self, key, ret) + +#define MultiDict_GetOne(self, key, ret) \ + MultiDictAPI->_MultiDict_GetOne(self, key, ret) + +#define MultiDict_Keys(self) MultiDictAPI->_MultiDict_Keys(self) + +#define MutliDict_Values(self) MultiDictAPI->_MultiDict_Values(self) + +#define MutliDict_Items(self) MultiDictAPI->_MultiDict_Items(self) + +#define MultiDict_Add(self, key, value) \ + MultiDictAPI->_MultiDict_Add(self, key, value) + +#define MultiDict_Clear(self) MultiDictAPI->_MultiDict_Clear(self) + +#define MultiDict_Extend(self, args, kwargs) \ + MultiDictAPI->_MultiDict_Extend(self, args, kwargs) + +#define MultiDict_Copy(self) MultiDictAPI->_MultiDict_Copy(self) + +#define MultiDict_SetDefault(self, key, value) \ + MultiDictAPI->_MultiDict_SetDefault(self, key, value) + +#define MultiDict_PopOne(self, key, ret) \ + MultiDictAPI->_MultiDict_PopOne(self, key, ret) + +#define MultiDict_PopItem(self) MultiDictAPI->_MultiDict_PopItem(self) + +#define MultiDict_Update(self, args, kwargs) \ + MultiDictAPI->_MultiDict_Update(self, args, kwargs) + +/* MultiDictProxy / CIMultiDictProxy Macros */ + +#define MultiDictProxy_GetAll(self, key, ret) \ + MultiDictAPI->_MultiDict_GetAll(self->md, key, ret) + +#define MultiDictProxy_GetOne(self, key, ret) \ + MultiDictAPI->_MultiDict_GetOne(self->md, key, ret) + +// NOTE: MultiDictProxy_Get will be going in the __init__.pxd file + +#define MultiDictProxy_Keys(self, key) \ + MultiDictAPI->_MultiDict_Keys(self->md, key) + +#define MutliDictProxy_Values(self) MultiDictAPI->_MultiDict_Values(self->md) + +#define MultiDictProxy_Items(self) MultiDictAPI->_MultiDict_Items(self->md) + +#define MultiDictProxy_Copy(self) MultiDictAPI->_MultiDict_Copy(self->md) + +#ifdef __cplusplus +} +#endif + +#endif // __CAPSULE_H__ diff --git a/requirements/ci.txt b/requirements/ci.txt index 1c7a96169..77ba3c951 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,2 +1,4 @@ -e . -r pytest.txt +# python capsule testing +cython==3.1.2 \ No newline at end of file diff --git a/tests/_multidict_cython.pyi b/tests/_multidict_cython.pyi new file mode 100644 index 000000000..da2464f20 --- /dev/null +++ b/tests/_multidict_cython.pyi @@ -0,0 +1,37 @@ +""" +This type stub file was generated by cyright. +With some minor edits... +""" + +from typing import Any, TypeVar + +from multidict import MultiDict, MultiDictProxy, CIMultiDict, CIMultiDictProxy, istr + +_T = TypeVar("_T") + +class Cython_MultiDict(MultiDict[_T]): + pass + +class Cython_MultiDictProxy(MultiDictProxy[_T]): + pass + +class Cython_CIMultiDict(CIMultiDict[_T]): + pass + +class Cython_CIMultiDictProxy(CIMultiDictProxy[_T]): + pass + +def multidict_create() -> MultiDict[_T]: ... +def cimultidict_create() -> CIMultiDict[_T]: ... +def multidictproxy_create(inner: object) -> MultiDictProxy[_T]: ... +def cimultidictproxy_create(inner: object) -> CIMultiDictProxy[_T]: ... +def multidict_add(md: MultiDict[_T]) -> None: ... +def multidict_update(md: MultiDict[_T], data: dict[str, Any]) -> None: ... +def multidict_copy(md: MultiDict[_T]) -> MultiDict[_T]: ... +def cimultidict_update(md: CIMultiDict[_T], data: dict[str, Any]) -> None: ... +def cimultidict_copy(md: CIMultiDict[_T]) -> CIMultiDict[_T]: ... +def istr_FromUnicode(data: str) -> istr: ... +def istr_check(data: object) -> bool: ... +def istr_checkexact(data: object) -> bool: ... + +class istrsubcls(istr): ... diff --git a/tests/_multidict_cython.pyx b/tests/_multidict_cython.pyx new file mode 100644 index 000000000..208d126f9 --- /dev/null +++ b/tests/_multidict_cython.pyx @@ -0,0 +1,59 @@ +# cython: language_level = 3 +# setuptools: include_dirs = MULTIDICT_HEADER_PATH + +from multidict cimport * +# Always remember to import_multidict or your script WILL FAIL +import_multidict() + +# NOTE: will use this check if the _multidict c module remained the same +Cython_MultiDict = MultiDict +Cython_MultiDictProxy = MultiDictProxy +Cython_CIMultiDict = CIMultiDict +Cython_CIMultiDictProxy = CIMultiDictProxy + + + + +def multidict_create(): + cdef MultiDict md = MultiDict() + return md + +def cimultidict_create(): + cdef CIMultiDict md = CIMultiDict() + return md + +def multidictproxy_create(object inner): + cdef MultiDictProxy md = MultiDictProxy(inner) # type: ignore + return md + +def cimultidictproxy_create(object inner): + cdef CIMultiDictProxy md = CIMultiDictProxy(inner) # type: ignore + return md + +def multidict_add(MultiDict md): + MultiDict_Add(md, "a", 1) + MultiDict_Add(md, "b", 2) + +def multidict_update(MultiDict md, *args, **kwargs): + MultiDict_Update(md, args, kwargs) + +def multidict_copy(MultiDict md): + return MultiDict_Copy(md) + +def multidict_popitem(MultiDict md): + return MultiDict_PopItem(md) + +def multidict_get(MultiDict md, str key): + return MultiDict_Get(md, key) + + +def istr_FromUnicode(str data): + return istr(data) # type: ignore + +def istr_check(object data): + return IStr_Check(data) + +def istr_checkexact(object data): + return IStr_CheckExact(data) + + diff --git a/tests/conftest.py b/tests/conftest.py index a37f58f2d..6ba7b66eb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,9 @@ from importlib import import_module from types import ModuleType from typing import Callable, Type, Union - +import subprocess +import os +import sys import pytest from multidict import ( @@ -176,6 +178,18 @@ def pytest_addoption( ) +def compile_cython_pycapsule_test() -> None: + """Allows pytest to compile cython before test starts""" + cmd = [] + if sys.prefix != sys.base_prefix: + if sys.platform == "win32": + cmd += [".venv\\Scripts\\activate.bat", "&&"] + else: + cmd += ["bash", ".venv/Scripts/activate.sh", ";"] + cmd += ["python", "tools/setup_cython_test.py", "build_ext", "--inplace"] + subprocess.run(cmd, env=os.environ, check=True, shell=True, capture_output=True) + + def pytest_collection_modifyitems( session: pytest.Session, config: pytest.Config, @@ -185,6 +199,7 @@ def pytest_collection_modifyitems( test_c_extensions = config.getoption("--c-extensions") is True if test_c_extensions: + compile_cython_pycapsule_test() return selected_tests: list[pytest.Item] = [] diff --git a/tests/test_cython_import.py b/tests/test_cython_import.py new file mode 100644 index 000000000..f6950b8c2 --- /dev/null +++ b/tests/test_cython_import.py @@ -0,0 +1,191 @@ +from dataclasses import dataclass +from functools import cache, cached_property +from importlib import import_module +from types import ModuleType +from typing import Callable, Type + +import pytest +import os +from multidict import MutableMultiMapping, istr + +skip_if_no_extensions = pytest.mark.skipif( + bool(os.environ.get("MULTIDICT_NO_EXTENSIONS")), reason="cython tests disabled" +) + + +@cache +def try_importing_c() -> ModuleType: + return import_module("multidict._multidict") + + +@cache +def try_impotring_cython() -> ModuleType: + return import_module("_multidict_cython") + + +@pytest.fixture(scope="module") +def c_module() -> ModuleType: + return try_importing_c() + + +@pytest.fixture(scope="module") +def cython_module() -> ModuleType: + return try_impotring_cython() + + +@dataclass(frozen=True) +class MultidictCythonImplementation: + """A facade for accessing importable multidict module variants. + + An instance essentially represents a c-extension or a pure-python module. + The actual underlying module is accessed dynamically through a property and + is cached. + + It also has a text tag depending on what variant it is, and a string + representation suitable for use in Pytest's test IDs via parametrization. + """ + + use_cython: bool + """A flag showing whether this is a pure-python module or a C-extension.""" + + @cached_property + def tag(self) -> str: + """Return a text representation of the pure-python attribute.""" + return "cython-extension" if self.use_cython else "c-extension" + + @cached_property + def imported_module(self) -> ModuleType: + """Return a loaded importable containing a multidict variant.""" + importable_module = ( + "_multidict_cython" if self.use_cython else "multidict._multidict" + ) + return import_module(f"{importable_module}") + + def __str__(self) -> str: + """Render the implementation facade instance as a string.""" + return f"{self.tag}-module" + + def get_class(self, attr: str) -> Type[MutableMultiMapping[str]]: + name = ("Cython_" + attr) if self.use_cython else attr + return self.imported_module.__dict__[name] # type: ignore[no-any-return] + + +@pytest.fixture( + scope="session", + params=( + pytest.param( + MultidictCythonImplementation(use_cython=False), + marks=pytest.mark.c_extension, + ), + pytest.param(MultidictCythonImplementation(use_cython=True)), + ), + ids=str, +) +def c_or_cython_multidict_implementation( + request: pytest.FixtureRequest, +) -> MultidictCythonImplementation: + return request.param # type: ignore[no-any-return] + + +@pytest.fixture(scope="session") +def any_multidict_c_or_cython_class( + c_or_cython_multidict_implementation: MultidictCythonImplementation, + any_multidict_class_name: str, +) -> Type[MutableMultiMapping[str]]: + return c_or_cython_multidict_implementation.get_class(any_multidict_class_name) + + +@pytest.fixture(scope="session") +def any_cython_md_creation_func( + any_multidict_class_name: str, +) -> Callable[[], MutableMultiMapping[str]]: + return getattr(try_impotring_cython(), any_multidict_class_name.lower() + "_create") # type: ignore[no-any-return] + + +@skip_if_no_extensions +def test_cython_types_are_equivilent_to_c( + cython_module: ModuleType, c_module: ModuleType +) -> None: + assert cython_module.Cython_MultiDict == c_module.MultiDict + assert cython_module.Cython_MultiDictProxy == c_module.MultiDictProxy + assert cython_module.Cython_CIMultiDict == c_module.CIMultiDict + assert cython_module.Cython_CIMultiDictProxy == c_module.CIMultiDictProxy + + +@skip_if_no_extensions +def test_cython_creation_of_multidict( + cython_module: ModuleType, + any_multidict_c_or_cython_class: Type[MutableMultiMapping[int]], +) -> None: + md: MutableMultiMapping[int] = cython_module.multidict_create() + md.add("a", 1) + b = any_multidict_c_or_cython_class() + b.add("a", 1) + assert md == b + + +@skip_if_no_extensions +def test_cython_addition( + any_cython_md_creation_func: Callable[[], MutableMultiMapping[str]], + cython_module: ModuleType, + any_multidict_c_or_cython_class: Type[MutableMultiMapping[str]], +) -> None: + md = any_cython_md_creation_func() + cython_module.multidict_add(md) + assert md == any_multidict_c_or_cython_class([("a", 1), ("b", 2)]) # type: ignore[call-arg] + + +@skip_if_no_extensions +def test_cython_update( + any_cython_md_creation_func: Callable[[], MutableMultiMapping[int]], + cython_module: ModuleType, + any_multidict_c_or_cython_class: Type[MutableMultiMapping[int]], +) -> None: + md = any_cython_md_creation_func() + cython_module.multidict_update(md, a=2, b=1) + assert md == any_multidict_c_or_cython_class([("a", 2), ("b", 1)]) # type: ignore[call-arg] + + +@skip_if_no_extensions +def test_cython_copy( + any_cython_md_creation_func: Callable[[], MutableMultiMapping[int]], + cython_module: ModuleType, + any_multidict_c_or_cython_class: Type[MutableMultiMapping[int]], +) -> None: + md = any_cython_md_creation_func() + cython_module.multidict_update(md, a=2, b=1) + new_md = cython_module.multidict_copy(md) + assert new_md == any_multidict_c_or_cython_class([("a", 2), ("b", 1)]) # type: ignore[call-arg] + + +@skip_if_no_extensions +def test_cython_get( + any_cython_md_creation_func: Callable[[], MutableMultiMapping[str]], + cython_module: ModuleType, +) -> None: + md = any_cython_md_creation_func() + cython_module.multidict_update(md, a=2, b=1) + assert md.get("b") == 1 # type:ignore[comparison-overlap] + assert cython_module.multidict_get(md, "a") == 2 + assert md.get("I DONT EXIST") is None + + # XXX: Broken, this sends a number when it should've been None, + # no clue why this happens + # assert cython_module.multidict_get(md, "I DONT EXIST!") + + +@skip_if_no_extensions +def test_istr_create(cython_module: ModuleType) -> None: + my_istr = cython_module.istr_FromUnicode("I-am-istr") + assert my_istr == "I-am-istr" + + +class istrsubcls(istr): + pass + + +@skip_if_no_extensions +def test_istr_checkexact(cython_module: ModuleType, c_module: ModuleType) -> None: + assert cython_module.istr_checkexact(c_module.istr("an istr")) + sub = istrsubcls("an istr") + assert not cython_module.istr_checkexact(sub), "subclassing should've raised false" diff --git a/tools/setup_cython_test.py b/tools/setup_cython_test.py new file mode 100644 index 000000000..195210d53 --- /dev/null +++ b/tools/setup_cython_test.py @@ -0,0 +1,17 @@ +from Cython.Build import cythonize +from setuptools import Extension, setup +import multidict + +# to compile run +# python tools/setup_cython_test.py build_ext --inplace +# NOTE: do not run in the tools directory directly + +if __name__ == "__main__": + setup( + ext_modules=cythonize( + Extension( + "tests._multidict_cython", sources=["tests/_multidict_cython.pyx"] + ) + ), + include_dirs=[multidict.get_include()], + )