From 17e651f6a4aa3983f2291fe97003cd043e630a10 Mon Sep 17 00:00:00 2001 From: Trevor Clinkenbeard Date: Thu, 2 Apr 2026 06:37:44 +0000 Subject: [PATCH 1/4] Split ISFreeNodes into separate sync and async functions --- flow/include/flow/IndexedSet.actor.h | 37 +++++++++++++++++++++++++--- flow/include/flow/IndexedSet.h | 4 +-- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/flow/include/flow/IndexedSet.actor.h b/flow/include/flow/IndexedSet.actor.h index 147786303cc..d4b12411f2b 100644 --- a/flow/include/flow/IndexedSet.actor.h +++ b/flow/include/flow/IndexedSet.actor.h @@ -32,10 +32,39 @@ #include "flow/Platform.h" #include "flow/actorcompiler.h" // This must be the last #include. +template +void ISFreeNodesSync(std::vector toFree) { + // Frees the forest of nodes in the 'toFree' vector without waiting. + + // Freeing many items from a large tree is bound by the memory latency to + // fetch each node from main memory. This code does a largely depth first + // traversal of the forest to be destroyed (using a stack) but prefetches + // each node and puts it on a short queue before actually processing it, so + // that several memory transactions can be outstanding simultaneously. + Deque prefetchQueue; + while (!prefetchQueue.empty() || !toFree.empty()) { + + while (prefetchQueue.size() < 10 && !toFree.empty()) { + _mm_prefetch((const char*)toFree.back(), _MM_HINT_T0); + prefetchQueue.push_back(toFree.back()); + toFree.pop_back(); + } + + auto n = prefetchQueue.front(); + prefetchQueue.pop_front(); + + if (n->child[0]) + toFree.push_back(n->child[0]); + if (n->child[1]) + toFree.push_back(n->child[1]); + n->child[0] = n->child[1] = 0; + delete n; + } +} + ACTOR template -[[flow_allow_discard]] Future ISFreeNodes(std::vector toFree, bool synchronous) { - // Frees the forest of nodes in the 'toFree' vector. - // If 'synchronous' is true, then there can be no waits. +[[flow_allow_discard]] Future ISFreeNodes(std::vector toFree) { + // Frees the forest of nodes in the 'toFree' vector, yielding periodically. state int eraseCount = 0; @@ -64,7 +93,7 @@ ACTOR template delete n; ++eraseCount; - if (!synchronous && eraseCount % 1000 == 0) + if (eraseCount % 1000 == 0) wait(yield()); } diff --git a/flow/include/flow/IndexedSet.h b/flow/include/flow/IndexedSet.h index 74daee5ffbc..967e4787f05 100644 --- a/flow/include/flow/IndexedSet.h +++ b/flow/include/flow/IndexedSet.h @@ -1392,7 +1392,7 @@ void IndexedSet::erase(typename IndexedSet::iterator begin std::vector::Node*> toFree; erase(begin, end, toFree); - ISFreeNodes(toFree, true); + ISFreeNodesSync(toFree); } template @@ -1407,7 +1407,7 @@ Future IndexedSet::eraseAsync(typename IndexedSet::i std::vector::Node*> toFree; erase(begin, end, toFree); - return uncancellable(ISFreeNodes(toFree, false)); + return uncancellable(ISFreeNodes(toFree)); } template From 4bccfd34e80c4da1e3d48dbf6af623abf450d7eb Mon Sep 17 00:00:00 2001 From: Trevor Clinkenbeard Date: Thu, 2 Apr 2026 06:41:28 +0000 Subject: [PATCH 2/4] Add ISFreeNodeImpl helper function --- flow/include/flow/IndexedSet.actor.h | 69 +++++++++++----------------- 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/flow/include/flow/IndexedSet.actor.h b/flow/include/flow/IndexedSet.actor.h index d4b12411f2b..fb283f4e26e 100644 --- a/flow/include/flow/IndexedSet.actor.h +++ b/flow/include/flow/IndexedSet.actor.h @@ -33,32 +33,41 @@ #include "flow/actorcompiler.h" // This must be the last #include. template -void ISFreeNodesSync(std::vector toFree) { - // Frees the forest of nodes in the 'toFree' vector without waiting. - +bool ISFreeNodeImpl(std::vector& toFree, Deque& prefetchQueue) { // Freeing many items from a large tree is bound by the memory latency to // fetch each node from main memory. This code does a largely depth first // traversal of the forest to be destroyed (using a stack) but prefetches // each node and puts it on a short queue before actually processing it, so // that several memory transactions can be outstanding simultaneously. - Deque prefetchQueue; - while (!prefetchQueue.empty() || !toFree.empty()) { + if (prefetchQueue.empty() && toFree.empty()) { + return false; + } + + while (prefetchQueue.size() < 10 && !toFree.empty()) { + _mm_prefetch((const char*)toFree.back(), _MM_HINT_T0); + prefetchQueue.push_back(toFree.back()); + toFree.pop_back(); + } + + auto n = prefetchQueue.front(); + prefetchQueue.pop_front(); - while (prefetchQueue.size() < 10 && !toFree.empty()) { - _mm_prefetch((const char*)toFree.back(), _MM_HINT_T0); - prefetchQueue.push_back(toFree.back()); - toFree.pop_back(); - } + if (n->child[0]) + toFree.push_back(n->child[0]); + if (n->child[1]) + toFree.push_back(n->child[1]); + n->child[0] = n->child[1] = 0; + delete n; - auto n = prefetchQueue.front(); - prefetchQueue.pop_front(); + return true; +} - if (n->child[0]) - toFree.push_back(n->child[0]); - if (n->child[1]) - toFree.push_back(n->child[1]); - n->child[0] = n->child[1] = 0; - delete n; +template +void ISFreeNodesSync(std::vector toFree) { + // Frees the forest of nodes in the 'toFree' vector without waiting. + + Deque prefetchQueue; + while (ISFreeNodeImpl(toFree, prefetchQueue)) { } } @@ -67,30 +76,8 @@ ACTOR template // Frees the forest of nodes in the 'toFree' vector, yielding periodically. state int eraseCount = 0; - - // Freeing many items from a large tree is bound by the memory latency to - // fetch each node from main memory. This code does a largely depth first - // traversal of the forest to be destroyed (using a stack) but prefetches - // each node and puts it on a short queue before actually processing it, so - // that several memory transactions can be outstanding simultaneously. state Deque prefetchQueue; - while (!prefetchQueue.empty() || !toFree.empty()) { - - while (prefetchQueue.size() < 10 && !toFree.empty()) { - _mm_prefetch((const char*)toFree.back(), _MM_HINT_T0); - prefetchQueue.push_back(toFree.back()); - toFree.pop_back(); - } - - auto n = prefetchQueue.front(); - prefetchQueue.pop_front(); - - if (n->child[0]) - toFree.push_back(n->child[0]); - if (n->child[1]) - toFree.push_back(n->child[1]); - n->child[0] = n->child[1] = 0; - delete n; + while (ISFreeNodeImpl(toFree, prefetchQueue)) { ++eraseCount; if (eraseCount % 1000 == 0) From fd7e406dd57b1c6b4197d76f3d9b218d29d8a7c8 Mon Sep 17 00:00:00 2001 From: Trevor Clinkenbeard Date: Thu, 2 Apr 2026 06:43:06 +0000 Subject: [PATCH 3/4] Convert ISFreeNodes to standard coroutine --- flow/include/flow/IndexedSet.actor.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/flow/include/flow/IndexedSet.actor.h b/flow/include/flow/IndexedSet.actor.h index fb283f4e26e..423a3b99935 100644 --- a/flow/include/flow/IndexedSet.actor.h +++ b/flow/include/flow/IndexedSet.actor.h @@ -71,8 +71,8 @@ void ISFreeNodesSync(std::vector toFree) { } } -ACTOR template -[[flow_allow_discard]] Future ISFreeNodes(std::vector toFree) { +template +Future ISFreeNodes(std::vector toFree) { // Frees the forest of nodes in the 'toFree' vector, yielding periodically. state int eraseCount = 0; @@ -80,11 +80,10 @@ ACTOR template while (ISFreeNodeImpl(toFree, prefetchQueue)) { ++eraseCount; - if (eraseCount % 1000 == 0) - wait(yield()); + if (eraseCount % 1000 == 0) { + co_await yield(); + } } - - return Void(); } #include "flow/unactorcompiler.h" From 9b472f22ae996d8bf403d9d9fdc684e1b1b5c142 Mon Sep 17 00:00:00 2001 From: Trevor Clinkenbeard Date: Thu, 2 Apr 2026 06:49:48 +0000 Subject: [PATCH 4/4] Move IndexedSet.actor.h contents to IndexedSet.h --- flow/include/flow/IndexedSet.actor.h | 90 ---------------------------- flow/include/flow/IndexedSet.h | 55 ++++++++++++++++- 2 files changed, 54 insertions(+), 91 deletions(-) delete mode 100644 flow/include/flow/IndexedSet.actor.h diff --git a/flow/include/flow/IndexedSet.actor.h b/flow/include/flow/IndexedSet.actor.h deleted file mode 100644 index 423a3b99935..00000000000 --- a/flow/include/flow/IndexedSet.actor.h +++ /dev/null @@ -1,90 +0,0 @@ -/* - * IndexedSet.actor.h - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source -// version. -#if defined(NO_INTELLISENSE) && !defined(FLOW_INDEXEDSET_ACTOR_G_H) -#define FLOW_INDEXEDSET_ACTOR_G_H -#include "flow/IndexedSet.actor.g.h" -#elif !defined(FLOW_INDEXEDSET_ACTOR_H) -#define FLOW_INDEXEDSET_ACTOR_H - -#include "flow/flow.h" -#include "flow/Platform.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -template -bool ISFreeNodeImpl(std::vector& toFree, Deque& prefetchQueue) { - // Freeing many items from a large tree is bound by the memory latency to - // fetch each node from main memory. This code does a largely depth first - // traversal of the forest to be destroyed (using a stack) but prefetches - // each node and puts it on a short queue before actually processing it, so - // that several memory transactions can be outstanding simultaneously. - if (prefetchQueue.empty() && toFree.empty()) { - return false; - } - - while (prefetchQueue.size() < 10 && !toFree.empty()) { - _mm_prefetch((const char*)toFree.back(), _MM_HINT_T0); - prefetchQueue.push_back(toFree.back()); - toFree.pop_back(); - } - - auto n = prefetchQueue.front(); - prefetchQueue.pop_front(); - - if (n->child[0]) - toFree.push_back(n->child[0]); - if (n->child[1]) - toFree.push_back(n->child[1]); - n->child[0] = n->child[1] = 0; - delete n; - - return true; -} - -template -void ISFreeNodesSync(std::vector toFree) { - // Frees the forest of nodes in the 'toFree' vector without waiting. - - Deque prefetchQueue; - while (ISFreeNodeImpl(toFree, prefetchQueue)) { - } -} - -template -Future ISFreeNodes(std::vector toFree) { - // Frees the forest of nodes in the 'toFree' vector, yielding periodically. - - state int eraseCount = 0; - state Deque prefetchQueue; - while (ISFreeNodeImpl(toFree, prefetchQueue)) { - ++eraseCount; - - if (eraseCount % 1000 == 0) { - co_await yield(); - } - } -} - -#include "flow/unactorcompiler.h" -#endif diff --git a/flow/include/flow/IndexedSet.h b/flow/include/flow/IndexedSet.h index 967e4787f05..3235cd70e05 100644 --- a/flow/include/flow/IndexedSet.h +++ b/flow/include/flow/IndexedSet.h @@ -1384,7 +1384,60 @@ Metric IndexedSet::sumTo(typename IndexedSet::const_iterat } #include "flow/flow.h" -#include "flow/IndexedSet.actor.h" + +template +bool ISFreeNodeImpl(std::vector& toFree, Deque& prefetchQueue) { + // Freeing many items from a large tree is bound by the memory latency to + // fetch each node from main memory. This code does a largely depth first + // traversal of the forest to be destroyed (using a stack) but prefetches + // each node and puts it on a short queue before actually processing it, so + // that several memory transactions can be outstanding simultaneously. + if (prefetchQueue.empty() && toFree.empty()) { + return false; + } + + while (prefetchQueue.size() < 10 && !toFree.empty()) { + _mm_prefetch((const char*)toFree.back(), _MM_HINT_T0); + prefetchQueue.push_back(toFree.back()); + toFree.pop_back(); + } + + auto n = prefetchQueue.front(); + prefetchQueue.pop_front(); + + if (n->child[0]) + toFree.push_back(n->child[0]); + if (n->child[1]) + toFree.push_back(n->child[1]); + n->child[0] = n->child[1] = 0; + delete n; + + return true; +} + +template +void ISFreeNodesSync(std::vector toFree) { + // Frees the forest of nodes in the 'toFree' vector without waiting. + + Deque prefetchQueue; + while (ISFreeNodeImpl(toFree, prefetchQueue)) { + } +} + +template +Future ISFreeNodes(std::vector toFree) { + // Frees the forest of nodes in the 'toFree' vector, yielding periodically. + + int eraseCount = 0; + Deque prefetchQueue; + while (ISFreeNodeImpl(toFree, prefetchQueue)) { + ++eraseCount; + + if (eraseCount % 1000 == 0) { + co_await yield(); + } + } +} template void IndexedSet::erase(typename IndexedSet::iterator begin,