From 4c539571277838190cf77501fe8ebc66a44b0c8f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 25 Jun 2025 11:33:08 +0200 Subject: [PATCH 1/7] Don't allocate new memory for htkeys on adding new item if the dict has deleted slots --- multidict/_multilib/hashtable.h | 43 ++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/multidict/_multilib/hashtable.h b/multidict/_multilib/hashtable.h index 53d39a58c..6add31fc6 100644 --- a/multidict/_multilib/hashtable.h +++ b/multidict/_multilib/hashtable.h @@ -209,7 +209,8 @@ _md_resize(MultiDictObject *md, uint8_t log2_newsize, bool update) } else { entry_t *new_ep = newentries; entry_t *old_ep = oldentries; - for (Py_ssize_t i = 0; i < oldkeys->nentries; ++i, ++old_ep) { + Py_ssize_t oldnumentries = oldkeys->nentries; + for (Py_ssize_t i = 0; i < oldnumentries; ++i, ++old_ep) { if (old_ep->identity != NULL) { *new_ep++ = *old_ep; } @@ -232,16 +233,52 @@ _md_resize(MultiDictObject *md, uint8_t log2_newsize, bool update) return 0; } +static inline int +_md_shrink(MultiDictObject *md, bool update) +{ + htkeys_t *keys = md->keys; + Py_ssize_t nentries = keys->nentries; + entry_t *entries = htkeys_entries(keys); + entry_t *new_ep = entries; + entry_t *old_ep = entries; + Py_ssize_t newnentries = nentries; + for (Py_ssize_t i = 0; i < nentries; ++i, ++old_ep) { + if (old_ep->identity != NULL) { + if (new_ep != old_ep) { + *new_ep++ = *old_ep; + } + } else { + newnentries -= 1; + } + } + keys->nentries = newnentries; + keys->usable += nentries - newnentries; + memset(&keys->indices[0], 0xff, ((size_t)1 << keys->log2_index_bytes)); + if (htkeys_build_indices(keys, entries, newnentries, update) < 0) { + return -1; + } + ASSERT_CONSISTENT(md, update); + return 0; +} + static inline int _md_resize_for_insert(MultiDictObject *md) { - return _md_resize(md, calculate_log2_keysize(GROWTH_RATE(md)), false); + if (md->used < md->keys->nentries) { + return _md_shrink(md, false); + } else { + return _md_resize(md, calculate_log2_keysize(GROWTH_RATE(md)), false); + } } static inline int _md_resize_for_update(MultiDictObject *md) { - return _md_resize(md, calculate_log2_keysize(GROWTH_RATE(md)), true); + if (md->used < md->keys->nentries) { + return _md_shrink(md, true); + } else { + return _md_resize(md, calculate_log2_keysize(GROWTH_RATE(md)), true); + } } static inline int From d195e25229d9d25ceb751e715bd88e49d711590b Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 25 Jun 2025 12:51:10 +0200 Subject: [PATCH 2/7] changelog --- CHANGES/1200.feature.rst | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 CHANGES/1200.feature.rst diff --git a/CHANGES/1200.feature.rst b/CHANGES/1200.feature.rst new file mode 100644 index 000000000..48d60ae06 --- /dev/null +++ b/CHANGES/1200.feature.rst @@ -0,0 +1,7 @@ +Skipped reallocation of internal htkeys structure on inserting new items if the +multidict has deleted items and it could be collapsed in-place. Removal of +``malloc()``/``free()`` could improve the performance a little. + +The change affects C implementation only, pure Python code is not changed. + +Patch by :user:`asvetlov`. From 43c1a4f393020ad5f15edddad35acabb6b80ad0d Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 25 Jun 2025 13:00:57 +0200 Subject: [PATCH 3/7] spelling --- docs/spelling_wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index a40068ed0..7eb712ab2 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -37,6 +37,7 @@ fallback fastpath filename formatters +htkeys gcc getitem github From 5a8485b7717800e87d7baa2a9b26d7a6ad24ab64 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 25 Jun 2025 14:22:36 +0200 Subject: [PATCH 4/7] Update CHANGES/1200.feature.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- CHANGES/1200.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES/1200.feature.rst b/CHANGES/1200.feature.rst index 48d60ae06..af8be8628 100644 --- a/CHANGES/1200.feature.rst +++ b/CHANGES/1200.feature.rst @@ -1,4 +1,4 @@ -Skipped reallocation of internal htkeys structure on inserting new items if the +Skipped reallocation of internal ``htkeys_t`` structure on inserting new items if the multidict has deleted items and it could be collapsed in-place. Removal of ``malloc()``/``free()`` could improve the performance a little. From 6b9ff209435db345c8707c65b23dce7ac53f4a4a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 25 Jun 2025 14:22:45 +0200 Subject: [PATCH 5/7] Update docs/spelling_wordlist.txt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- docs/spelling_wordlist.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 7eb712ab2..a40068ed0 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -37,7 +37,6 @@ fallback fastpath filename formatters -htkeys gcc getitem github From 06717cbe637d722201994c7a18888186e9a1568e Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 25 Jun 2025 14:26:04 +0200 Subject: [PATCH 6/7] Update CHANGES/1200.feature.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- CHANGES/1200.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES/1200.feature.rst b/CHANGES/1200.feature.rst index af8be8628..95dbef1d4 100644 --- a/CHANGES/1200.feature.rst +++ b/CHANGES/1200.feature.rst @@ -1,4 +1,4 @@ -Skipped reallocation of internal ``htkeys_t`` structure on inserting new items if the +Stopped reallocating memory for the internal ``htkeys_t`` structure when inserting new items if the multidict has deleted items and it could be collapsed in-place. Removal of ``malloc()``/``free()`` could improve the performance a little. From 1e6c2bc2aaac7ba32222b510ce437fe7bfc48b5c Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 25 Jun 2025 14:26:15 +0200 Subject: [PATCH 7/7] Update CHANGES/1200.feature.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- CHANGES/1200.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES/1200.feature.rst b/CHANGES/1200.feature.rst index 95dbef1d4..0f7463bc8 100644 --- a/CHANGES/1200.feature.rst +++ b/CHANGES/1200.feature.rst @@ -1,6 +1,6 @@ Stopped reallocating memory for the internal ``htkeys_t`` structure when inserting new items if the multidict has deleted items and it could be collapsed in-place. Removal of -``malloc()``/``free()`` could improve the performance a little. +``malloc()``/``free()`` improves the performance slightly. The change affects C implementation only, pure Python code is not changed.