feat: hashlib compatibility

- algorithms_available / algorithms_guaranteed: module attributes
- Str rejection: TypeError('Strings must be encoded before hashing')
- data= keyword argument (renamed from input=)
- Buffer types: bytes, bytearray, memoryview, array, mmap,
  PickleBuffer, ctypes
- 13 hashlib compat tests + 11 buffer input tests
- 3.8.0.dev5

Author: ifduyue

src/_xxhash.c

@@ -45,28 +45,18 @@
 #define XXH128_BLOCKSIZE 64
 
 
-/* Get a buffer from an object, or UTF-8 encode if it's a str.
- * On success, *owner is set to the object that owns the buffer
- * (NULL if the arg itself supports the buffer protocol).
- * Caller must PyBuffer_Release(buf) and Py_XDECREF(*owner). */
+/* Get a buffer from an object. Rejects str with hashlib-compatible error. */
 #ifndef Py_ALWAYS_INLINE
 #  define Py_ALWAYS_INLINE
 #endif
 
 static Py_ALWAYS_INLINE int
 _get_buffer_or_str(PyObject *obj, Py_buffer *buf, PyObject **owner)
 {
-    /* Check str first to avoid a guaranteed-failing PyObject_GetBuffer call
-     * and the resulting set/clear of a TypeError. */
     if (PyUnicode_Check(obj)) {
-        *owner = PyUnicode_AsUTF8String(obj);
-        if (*owner == NULL)
-            return -1;
-        if (PyObject_GetBuffer(*owner, buf, PyBUF_SIMPLE) < 0) {
-            Py_DECREF(*owner);
-            return -1;
-        }
-        return 0;
+        PyErr_SetString(PyExc_TypeError,
+            "Strings must be encoded before hashing");
+        return -1;
     }
     if (PyObject_GetBuffer(obj, buf, PyBUF_SIMPLE) < 0)
         return -1;
@@ -75,7 +65,7 @@ _get_buffer_or_str(PyObject *obj, Py_buffer *buf, PyObject **owner)
 }
 
 /* Parse input buffer and optional seed from fastcall arguments.
- * Handles: positional 'input', positional 'seed', keyword 'input',
+ * Handles: positional 'data', positional 'seed', keyword 'data',
  * keyword 'seed', with proper error reporting for unknown keywords,
  * duplicate arguments, and too many positional args.
  * Returns 0 on success, -1 on error with exception set. */
@@ -120,10 +110,10 @@ _parse_fastcall_args(PyObject *const *args, Py_ssize_t nargs,
             PyObject *key = PyTuple_GET_ITEM(kwnames, i);
             PyObject *val = args[nargs + i];
 
-            if (PyUnicode_CompareWithASCIIString(key, "input") == 0) {
+            if (PyUnicode_CompareWithASCIIString(key, "data") == 0) {
                 if (input_found) {
                     PyErr_Format(PyExc_TypeError,
-                        "%s() got multiple values for argument 'input'",
+                        "%s() got multiple values for argument 'data'",
                         funcname);
                     goto error;
                 }
@@ -152,7 +142,7 @@ _parse_fastcall_args(PyObject *const *args, Py_ssize_t nargs,
 
     if (!input_found && input_required) {
         PyErr_Format(PyExc_TypeError,
-            "%s() missing required argument 'input'", funcname);
+            "%s() missing required argument 'data'", funcname);
         return -1;
     }
     return 0;
@@ -559,7 +549,7 @@ static PyObject *PYXXH32_new(PyTypeObject *type, PyObject *args, PyObject *kwarg
 static int PYXXH32_init(PYXXH32Object *self, PyObject *args, PyObject *kwargs)
 {
     XXH32_hash_t seed = 0;
-    char *keywords[] = {"input", "seed", NULL};
+    char *keywords[] = {"data", "seed", NULL};
     Py_buffer buf;
 
     buf.buf = buf.obj = NULL;
@@ -580,8 +570,8 @@ static int PYXXH32_init(PYXXH32Object *self, PyObject *args, PyObject *kwargs)
 
 PyDoc_STRVAR(
     PYXXH32_update_doc,
-    "update (input)\n\n"
-    "Update the xxh32 object with the string input. Repeated calls are\n"
+    "update (data)\n\n"
+    "Update the xxh32 object with the string data. Repeated calls are\n"
     "equivalent to a single call with the concatenation of all the arguments.");
 
 static PyObject *PYXXH32_update(PYXXH32Object *self, PyObject *args)
@@ -768,7 +758,7 @@ PyDoc_STRVAR(
     "\n"
     "Methods:\n"
     "\n"
-    "update(input) -- updates the current digest with the provided string.\n"
+    "update(data) -- updates the current digest with the provided data.\n"
     "digest() -- return the current digest value\n"
     "hexdigest() -- return the current digest as a string of hexadecimal digits\n"
     "intdigest() -- return the current digest as an integer\n"
@@ -915,7 +905,7 @@ static PyObject *PYXXH64_new(PyTypeObject *type, PyObject *args, PyObject *kwarg
 static int PYXXH64_init(PYXXH64Object *self, PyObject *args, PyObject *kwargs)
 {
     XXH64_hash_t seed = 0;
-    char *keywords[] = {"input", "seed", NULL};
+    char *keywords[] = {"data", "seed", NULL};
     Py_buffer buf;
 
     buf.buf = buf.obj = NULL;
@@ -936,8 +926,8 @@ static int PYXXH64_init(PYXXH64Object *self, PyObject *args, PyObject *kwargs)
 
 PyDoc_STRVAR(
     PYXXH64_update_doc,
-    "update (input)\n\n"
-    "Update the xxh64 object with the string input. Repeated calls are\n"
+    "update (data)\n\n"
+    "Update the xxh64 object with the string data. Repeated calls are\n"
     "equivalent to a single call with the concatenation of all the arguments.");
 
 static PyObject *PYXXH64_update(PYXXH64Object *self, PyObject *args)
@@ -1124,7 +1114,7 @@ PyDoc_STRVAR(
     "\n"
     "Methods:\n"
     "\n"
-    "update(input) -- updates the current digest with an additional string\n"
+    "update(data) -- updates the current digest with additional data\n"
     "digest() -- return the current digest value\n"
     "hexdigest() -- return the current digest as a string of hexadecimal digits\n"
     "intdigest() -- return the current digest as an integer\n"
@@ -1270,7 +1260,7 @@ static PyObject *PYXXH3_64_new(PyTypeObject *type, PyObject *args, PyObject *kwa
 static int PYXXH3_64_init(PYXXH3_64Object *self, PyObject *args, PyObject *kwargs)
 {
     XXH64_hash_t seed = 0;
-    char *keywords[] = {"input", "seed", NULL};
+    char *keywords[] = {"data", "seed", NULL};
     Py_buffer buf;
 
     buf.buf = buf.obj = NULL;
@@ -1291,8 +1281,8 @@ static int PYXXH3_64_init(PYXXH3_64Object *self, PyObject *args, PyObject *kwarg
 
 PyDoc_STRVAR(
     PYXXH3_64_update_doc,
-    "update (input)\n\n"
-    "Update the xxh3_64 object with the string input. Repeated calls are\n"
+    "update (data)\n\n"
+    "Update the xxh3_64 object with the string data. Repeated calls are\n"
     "equivalent to a single call with the concatenation of all the arguments.");
 
 static PyObject *PYXXH3_64_update(PYXXH3_64Object *self, PyObject *args)
@@ -1487,7 +1477,7 @@ PyDoc_STRVAR(
     "\n"
     "Methods:\n"
     "\n"
-    "update(input) -- updates the current digest with an additional string\n"
+    "update(data) -- updates the current digest with additional data\n"
     "digest() -- return the current digest value\n"
     "hexdigest() -- return the current digest as a string of hexadecimal digits\n"
     "intdigest() -- return the current digest as an integer\n"
@@ -1634,7 +1624,7 @@ static PyObject *PYXXH3_128_new(PyTypeObject *type, PyObject *args, PyObject *kw
 static int PYXXH3_128_init(PYXXH3_128Object *self, PyObject *args, PyObject *kwargs)
 {
     XXH64_hash_t seed = 0;
-    char *keywords[] = {"input", "seed", NULL};
+    char *keywords[] = {"data", "seed", NULL};
     Py_buffer buf;
 
     buf.buf = buf.obj = NULL;
@@ -1655,8 +1645,8 @@ static int PYXXH3_128_init(PYXXH3_128Object *self, PyObject *args, PyObject *kwa
 
 PyDoc_STRVAR(
     PYXXH3_128_update_doc,
-    "update (input)\n\n"
-    "Update the xxh3_128 object with the string input. Repeated calls are\n"
+    "update (data)\n\n"
+    "Update the xxh3_128 object with the string data. Repeated calls are\n"
     "equivalent to a single call with the concatenation of all the arguments.");
 
 static PyObject *PYXXH3_128_update(PYXXH3_128Object *self, PyObject *args)
@@ -1868,7 +1858,7 @@ PyDoc_STRVAR(
     "\n"
     "Methods:\n"
     "\n"
-    "update(input) -- updates the current digest with an additional string\n"
+    "update(data) -- updates the current digest with additional data\n"
     "digest() -- return the current digest value\n"
     "hexdigest() -- return the current digest as a string of hexadecimal digits\n"
     "intdigest() -- return the current digest as an integer\n"

tests/test_benchmark.py

@@ -90,42 +90,17 @@ def test_xxh64_hexdigest_5b():
     xxhash.xxh64_hexdigest(DATA_5B)
 
 
-# ── str input (tests _get_buffer_or_str UTF-8 encoding path) ────────
-
-DATA_STR = "hello world"
-
-
-@pytest.mark.benchmark
-def test_xxh32_intdigest_str():
-    xxhash.xxh32_intdigest(DATA_STR)
-
-
-@pytest.mark.benchmark
-def test_xxh64_intdigest_str():
-    xxhash.xxh64_intdigest(DATA_STR)
-
-
-@pytest.mark.benchmark
-def test_xxh3_64_intdigest_str():
-    xxhash.xxh3_64_intdigest(DATA_STR)
-
-
-@pytest.mark.benchmark
-def test_xxh3_128_intdigest_str():
-    xxhash.xxh3_128_intdigest(DATA_STR)
-
-
 # ── type constructor (tests tp_vectorcall) ──────────────────────────
 
 
 @pytest.mark.benchmark
 def test_xxh32_ctor():
-    xxhash.xxh32(DATA_STR)
+    xxhash.xxh32(DATA_5B)
 
 
 @pytest.mark.benchmark
 def test_xxh32_ctor_seed():
-    xxhash.xxh32(DATA_STR, seed=SEED_32)
+    xxhash.xxh32(DATA_5B, seed=SEED_32)
 
 
 @pytest.mark.benchmark
@@ -135,17 +110,17 @@ def test_xxh32_ctor_empty():
 
 @pytest.mark.benchmark
 def test_xxh64_ctor():
-    xxhash.xxh64(DATA_STR, seed=SEED_64)
+    xxhash.xxh64(DATA_5B, seed=SEED_64)
 
 
 @pytest.mark.benchmark
 def test_xxh3_64_ctor():
-    xxhash.xxh3_64(DATA_STR, seed=SEED_64)
+    xxhash.xxh3_64(DATA_5B, seed=SEED_64)
 
 
 @pytest.mark.benchmark
 def test_xxh3_128_ctor():
-    xxhash.xxh3_128(DATA_STR, seed=SEED_64)
+    xxhash.xxh3_128(DATA_5B, seed=SEED_64)
 
 
 # ── 2MB throughput: hashing dominates, call overhead negligible ─────

tests/test_fastcall.py

@@ -36,9 +36,12 @@ def test_input_bytes(self):
             self._check(a, self.data)
 
     def test_input_str(self):
+        """hashlib compatibility: str raises TypeError."""
         s = self.data.decode()
         for a in self.algorithms:
-            self._check(a, s)
+            for fn in self._funcs(a):
+                with self.assertRaises(TypeError):
+                    fn(s)
 
     def test_input_empty(self):
         for a in self.algorithms:
@@ -64,13 +67,13 @@ def test_positional_seed_xxh3_128(self):
 
     # ── keyword input ─────────────────────────────────────────────
 
-    def test_keyword_input(self):
+    def test_keyword_data(self):
         for a in self.algorithms:
-            self._check(a, input=self.data)
+            self._check(a, data=self.data)
 
-    def test_keyword_input_and_seed(self):
+    def test_keyword_data_and_seed(self):
         for a in self.algorithms:
-            self._check(a, input=self.data, seed=42)
+            self._check(a, data=self.data, seed=42)
 
     # ── keyword seed (with positional input) ──────────────────────
 
@@ -105,6 +108,33 @@ def test_input_array(self):
         for a in self.algorithms:
             self._check(a, array.array('B', self.data))
 
+    def test_input_mmap(self):
+        import mmap, tempfile, os
+        with tempfile.NamedTemporaryFile(delete=False) as f:
+            f.write(self.data)
+            f.flush()
+        try:
+            with open(f.name, 'rb') as f2:
+                with mmap.mmap(f2.fileno(), 0, access=mmap.ACCESS_READ) as m:
+                    for a in self.algorithms:
+                        self._check(a, m)
+        finally:
+            os.unlink(f.name)
+
+    def test_input_pickle_buffer(self):
+        try:
+            from pickle import PickleBuffer
+        except ImportError:
+            raise self.skipTest('PickleBuffer not available')
+        for a in self.algorithms:
+            self._check(a, PickleBuffer(self.data))
+
+    def test_input_ctypes(self):
+        import ctypes
+        buf = (ctypes.c_char * len(self.data)).from_buffer_copy(self.data)
+        for a in self.algorithms:
+            self._check(a, buf)
+
 
 class TestFastcallErrors(unittest.TestCase):
     """Invalid argument passing: all error cases."""
@@ -141,16 +171,17 @@ def test_too_many_positional(self):
 
     # ── unknown keyword ───────────────────────────────────────────
 
-    def test_unknown_keyword(self):
-        self._assert_all_raise(TypeError, self.data, bad=1)
+    def test_unknown_keyword_input(self):
+        """Old 'input' keyword is now unknown — was renamed to 'data'."""
+        self._assert_all_raise(TypeError, input=self.data)
 
-    def test_unknown_keyword_input_kw(self):
-        self._assert_all_raise(TypeError, input=self.data, bad=1)
+    def test_unknown_keyword_data_kw(self):
+        self._assert_all_raise(TypeError, data=self.data, bad=1)
 
     # ── duplicate arguments ───────────────────────────────────────
 
     def test_duplicate_input(self):
-        self._assert_all_raise(TypeError, self.data, input=self.data)
+        self._assert_all_raise(TypeError, self.data, data=self.data)
 
     def test_duplicate_seed(self):
         self._assert_all_raise(TypeError, self.data, 0, seed=1)
@@ -164,15 +195,15 @@ def test_invalid_seed_keyword(self):
         self._assert_all_raise(TypeError, self.data, seed='bad')
 
     def test_invalid_seed_with_input_kw(self):
-        self._assert_all_raise(TypeError, input=self.data, seed='bad')
+        self._assert_all_raise(TypeError, data=self.data, seed='bad')
 
     # ── invalid input type (not str, not buffer) ──────────────────
 
     def test_input_not_bytes_or_str(self):
         self._assert_all_raise(TypeError, 12345)
 
     def test_input_not_bytes_or_str_kw(self):
-        self._assert_all_raise(TypeError, input=12345)
+        self._assert_all_raise(TypeError, data=12345)
 
 
 class TestFastcallSeedOverflow(unittest.TestCase):