Skip to content

Commit c49ccf1

Browse files
committed
Add continuation_op to eliminate heap allocation on executor post
continuation_op wraps a capy::continuation inside a scheduler_op so the scheduler can queue it in the same FIFO as I/O completions without heap-allocating a post_handler. A magic tag allows try_from_continuation() to distinguish tagged continuations from bare ones (e.g. capy's run_async), falling back to the existing heap path when needed. All sites that previously stored capy::continuation now store continuation_op, giving the zero-alloc path to timer completions, signal delivery, and tcp_server worker management. The tcp_server launcher and test helpers still use the heap path since their continuations are stack-local
1 parent d2c0e86 commit c49ccf1

22 files changed

+230
-96
lines changed
Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
//
2+
// Copyright (c) 2026 Michael Vandeberg
3+
//
4+
// Distributed under the Boost Software License, Version 1.0. (See accompanying
5+
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6+
//
7+
// Official repository: https://github.com/cppalliance/corosio
8+
//
9+
10+
#ifndef BOOST_COROSIO_DETAIL_CONTINUATION_OP_HPP
11+
#define BOOST_COROSIO_DETAIL_CONTINUATION_OP_HPP
12+
13+
#include <boost/corosio/detail/scheduler_op.hpp>
14+
#include <boost/capy/continuation.hpp>
15+
16+
#include <atomic>
17+
#include <cstdint>
18+
#include <cstring>
19+
20+
namespace boost::corosio::detail {
21+
22+
/* Scheduler operation that resumes a capy::continuation.
23+
24+
Embeds a continuation alongside a scheduler_op so the
25+
scheduler can queue it in the same FIFO as I/O completions
26+
without a heap allocation. The continuation lives in the
27+
caller's coroutine frame (awaitable or op struct); this
28+
wrapper gives it a scheduler_op identity.
29+
30+
io_context::executor_type::post(continuation&) uses
31+
try_from_continuation() to recover the enclosing
32+
continuation_op via a magic tag. The tag is read through
33+
memcpy (not through a continuation_op*) so that UBSan
34+
does not flag the speculative pointer arithmetic when the
35+
continuation is not actually inside a continuation_op.
36+
*/
37+
struct continuation_op final : scheduler_op
38+
{
39+
static constexpr std::uint32_t magic_ = 0xC0710Au;
40+
41+
std::uint32_t tag_ = magic_;
42+
capy::continuation cont;
43+
44+
continuation_op() noexcept : scheduler_op(&do_complete) {}
45+
46+
// Reactor backends (epoll, select, kqueue) dispatch through
47+
// virtual operator()(). IOCP dispatches through func_ which
48+
// routes to do_complete below.
49+
void operator()() override
50+
{
51+
std::atomic_thread_fence(std::memory_order_acquire);
52+
cont.h.resume();
53+
}
54+
55+
void destroy() override
56+
{
57+
if (cont.h)
58+
cont.h.destroy();
59+
}
60+
61+
private:
62+
// IOCP completion entry point. owner == nullptr means destroy.
63+
static void do_complete(
64+
void* owner,
65+
scheduler_op* base,
66+
std::uint32_t,
67+
std::uint32_t)
68+
{
69+
auto* self = static_cast<continuation_op*>(base);
70+
if (!owner)
71+
{
72+
if (self->cont.h)
73+
self->cont.h.destroy();
74+
return;
75+
}
76+
std::atomic_thread_fence(std::memory_order_acquire);
77+
self->cont.h.resume();
78+
}
79+
80+
public:
81+
82+
// Recover the enclosing continuation_op from its cont member.
83+
// Returns nullptr if the continuation is not tagged (bare
84+
// capy::continuation from capy internals like run_async).
85+
static continuation_op* try_from_continuation(
86+
capy::continuation& c) noexcept
87+
{
88+
// offsetof on non-standard-layout is conditionally-supported;
89+
// suppress the warning — all targeted compilers handle this
90+
// correctly and the self-relative arithmetic is move-safe.
91+
#if defined(__GNUC__) || defined(__clang__)
92+
#pragma GCC diagnostic push
93+
#pragma GCC diagnostic ignored "-Winvalid-offsetof"
94+
#endif
95+
constexpr auto cont_off = offsetof(continuation_op, cont);
96+
constexpr auto tag_off = offsetof(continuation_op, tag_);
97+
#if defined(__GNUC__) || defined(__clang__)
98+
#pragma GCC diagnostic pop
99+
#endif
100+
// Read the tag through memcpy from a char*, not through a
101+
// continuation_op*. This avoids UBSan's vptr check when
102+
// the continuation is not actually inside a continuation_op.
103+
auto* base = reinterpret_cast<char*>(&c) - cont_off;
104+
std::uint32_t tag;
105+
std::memcpy(&tag, base + tag_off, sizeof(tag));
106+
if (tag != magic_)
107+
return nullptr;
108+
return reinterpret_cast<continuation_op*>(base);
109+
}
110+
};
111+
112+
} // namespace boost::corosio::detail
113+
114+
#endif

include/boost/corosio/io/io_timer.hpp

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@
1212
#define BOOST_COROSIO_IO_IO_TIMER_HPP
1313

1414
#include <boost/corosio/detail/config.hpp>
15+
#include <boost/corosio/detail/continuation_op.hpp>
1516
#include <boost/corosio/io/io_object.hpp>
16-
#include <boost/capy/continuation.hpp>
1717
#include <boost/capy/io_result.hpp>
1818
#include <boost/capy/error.hpp>
1919
#include <boost/capy/ex/executor_ref.hpp>
@@ -47,7 +47,7 @@ class BOOST_COROSIO_DECL io_timer : public io_object
4747
io_timer& t_;
4848
std::stop_token token_;
4949
mutable std::error_code ec_;
50-
capy::continuation cont_;
50+
detail::continuation_op cont_op_;
5151

5252
explicit wait_awaitable(io_timer& t) noexcept : t_(t) {}
5353

@@ -67,7 +67,7 @@ class BOOST_COROSIO_DECL io_timer : public io_object
6767
-> std::coroutine_handle<>
6868
{
6969
token_ = env->stop_token;
70-
cont_.h = h;
70+
cont_op_.cont.h = h;
7171
auto& impl = t_.get();
7272
// Inline fast path: already expired and not in the heap
7373
if (impl.heap_index_ == implementation::npos &&
@@ -78,10 +78,10 @@ class BOOST_COROSIO_DECL io_timer : public io_object
7878
token_ = {}; // match normal path so await_resume
7979
// returns ec_, not a stale stop check
8080
auto d = env->executor;
81-
d.post(cont_);
81+
d.post(cont_op_.cont);
8282
return std::noop_coroutine();
8383
}
84-
return impl.wait(h, env->executor, std::move(token_), &ec_, &cont_);
84+
return impl.wait(h, env->executor, std::move(token_), &ec_, &cont_op_.cont);
8585
}
8686
};
8787

include/boost/corosio/io_context.hpp

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#define BOOST_COROSIO_IO_CONTEXT_HPP
1414

1515
#include <boost/corosio/detail/config.hpp>
16+
#include <boost/corosio/detail/continuation_op.hpp>
1617
#include <boost/corosio/detail/platform.hpp>
1718
#include <boost/corosio/detail/scheduler.hpp>
1819
#include <boost/capy/continuation.hpp>
@@ -379,31 +380,50 @@ class io_context::executor_type
379380
/** Dispatch a continuation.
380381
381382
Returns a handle for symmetric transfer. If called from
382-
within `run()`, returns `c.h`. Otherwise posts the coroutine
383-
for later execution and returns `std::noop_coroutine()`.
383+
within `run()`, returns `c.h`. Otherwise posts the
384+
enclosing continuation_op as a scheduler_op for later
385+
execution and returns `std::noop_coroutine()`.
384386
385-
@param c The continuation to dispatch.
387+
@param c The continuation to dispatch. Must be the `cont`
388+
member of a `detail::continuation_op`.
386389
387390
@return A handle for symmetric transfer or `std::noop_coroutine()`.
388391
*/
389392
std::coroutine_handle<> dispatch(capy::continuation& c) const
390393
{
391394
if (running_in_this_thread())
392395
return c.h;
393-
ctx_->sched_->post(c.h);
396+
post(c);
394397
return std::noop_coroutine();
395398
}
396399

397400
/** Post a continuation for deferred execution.
398401
399-
The coroutine will be resumed during a subsequent call to
400-
`run()`.
401-
402-
@param c The continuation to post.
402+
If the continuation is backed by a continuation_op
403+
(tagged), posts it directly as a scheduler_op — zero
404+
heap allocation. Otherwise falls back to the
405+
heap-allocating post(coroutine_handle<>) path.
403406
*/
404407
void post(capy::continuation& c) const
405408
{
406-
ctx_->sched_->post(c.h);
409+
auto* op = detail::continuation_op::try_from_continuation(c);
410+
if (op)
411+
ctx_->sched_->post(op);
412+
else
413+
ctx_->sched_->post(c.h);
414+
}
415+
416+
/** Post a bare coroutine handle for deferred execution.
417+
418+
Heap-allocates a scheduler_op to wrap the handle. Prefer
419+
posting through a continuation_op-backed continuation when
420+
the continuation has suitable lifetime.
421+
422+
@param h The coroutine handle to post.
423+
*/
424+
void post(std::coroutine_handle<> h) const
425+
{
426+
ctx_->sched_->post(h);
407427
}
408428

409429
/** Compare two executors for equality.

include/boost/corosio/native/detail/epoll/epoll_tcp_acceptor_service.hpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -177,8 +177,8 @@ epoll_tcp_acceptor::accept(
177177
if (impl_out)
178178
*impl_out = nullptr;
179179
}
180-
op.cont.h = h;
181-
return dispatch_coro(ex, op.cont);
180+
op.cont_op.cont.h = h;
181+
return dispatch_coro(ex, op.cont_op.cont);
182182
}
183183

184184
op.accepted_fd = accepted;

include/boost/corosio/native/detail/iocp/win_overlapped_op.hpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ struct overlapped_op
6262

6363
long ready_ = 0;
6464
std::coroutine_handle<> h;
65-
capy::continuation cont;
65+
detail::continuation_op cont_op;
6666
capy::executor_ref ex;
6767
std::error_code* ec_out = nullptr;
6868
std::size_t* bytes_out = nullptr;
@@ -145,8 +145,8 @@ struct overlapped_op
145145
if (bytes_out)
146146
*bytes_out = static_cast<std::size_t>(bytes_transferred);
147147

148-
cont.h = h;
149-
dispatch_coro(ex, cont).resume();
148+
cont_op.cont.h = h;
149+
dispatch_coro(ex, cont_op.cont).resume();
150150
}
151151

152152
/** Disarm cancellation and abandon the coroutine handle. */

include/boost/corosio/native/detail/iocp/win_resolver_service.hpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -270,8 +270,8 @@ resolve_op::do_complete(
270270

271271
op->cancel_handle = nullptr;
272272

273-
op->cont.h = op->h;
274-
dispatch_coro(op->ex, op->cont).resume();
273+
op->cont_op.cont.h = op->h;
274+
dispatch_coro(op->ex, op->cont_op.cont).resume();
275275
}
276276

277277
// reverse_resolve_op
@@ -315,8 +315,8 @@ reverse_resolve_op::do_complete(
315315
op->ep, std::move(op->stored_host), std::move(op->stored_service));
316316
}
317317

318-
op->cont.h = op->h;
319-
dispatch_coro(op->ex, op->cont).resume();
318+
op->cont_op.cont.h = op->h;
319+
dispatch_coro(op->ex, op->cont_op.cont).resume();
320320
}
321321

322322
// win_resolver

include/boost/corosio/native/detail/iocp/win_signal.hpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
#include <boost/corosio/detail/config.hpp>
1919
#include <boost/corosio/signal_set.hpp>
20+
#include <boost/corosio/detail/continuation_op.hpp>
2021
#include <boost/corosio/detail/intrusive.hpp>
2122
#include <boost/corosio/detail/scheduler_op.hpp>
2223
#include <boost/capy/continuation.hpp>
@@ -44,7 +45,7 @@ enum
4445
struct signal_op : scheduler_op
4546
{
4647
std::coroutine_handle<> h;
47-
capy::continuation cont;
48+
detail::continuation_op cont_op;
4849
capy::executor_ref d;
4950
std::error_code* ec_out = nullptr;
5051
int* signal_out = nullptr;

include/boost/corosio/native/detail/iocp/win_signals.hpp

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -304,8 +304,8 @@ signal_op::do_complete(
304304
auto* service = op->svc;
305305
op->svc = nullptr;
306306

307-
op->cont.h = op->h;
308-
dispatch_coro(op->d, op->cont).resume();
307+
op->cont_op.cont.h = op->h;
308+
dispatch_coro(op->d, op->cont_op.cont).resume();
309309

310310
if (service)
311311
service->work_finished();
@@ -338,8 +338,8 @@ win_signal::wait(
338338
*ec = make_error_code(capy::error::canceled);
339339
if (signal_out)
340340
*signal_out = 0;
341-
pending_op_.cont.h = h;
342-
dispatch_coro(d, pending_op_.cont).resume();
341+
pending_op_.cont_op.cont.h = h;
342+
dispatch_coro(d, pending_op_.cont_op.cont).resume();
343343
// completion is always posted to scheduler queue, never inline.
344344
return std::noop_coroutine();
345345
}
@@ -612,8 +612,8 @@ win_signals::cancel_wait(win_signal& impl)
612612
*op->ec_out = make_error_code(capy::error::canceled);
613613
if (op->signal_out)
614614
*op->signal_out = 0;
615-
op->cont.h = op->h;
616-
dispatch_coro(op->d, op->cont).resume();
615+
op->cont_op.cont.h = op->h;
616+
dispatch_coro(op->d, op->cont_op.cont).resume();
617617
sched_.work_finished();
618618
}
619619
}

include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -238,11 +238,11 @@ accept_op::do_complete(
238238
*op->impl_out = nullptr;
239239
}
240240

241-
op->cont.h = op->h;
241+
op->cont_op.cont.h = op->h;
242242
auto saved_ex = op->ex;
243243
auto prevent_premature_destruction = std::move(op->acceptor_ptr);
244244

245-
dispatch_coro(saved_ex, op->cont).resume();
245+
dispatch_coro(saved_ex, op->cont_op.cont).resume();
246246
}
247247

248248
// connect_op completion handler

include/boost/corosio/native/detail/kqueue/kqueue_tcp_acceptor_service.hpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -208,8 +208,8 @@ kqueue_tcp_acceptor::accept(
208208
if (impl_out)
209209
*impl_out = nullptr;
210210
}
211-
op.cont.h = h;
212-
return dispatch_coro(ex, op.cont);
211+
op.cont_op.cont.h = h;
212+
return dispatch_coro(ex, op.cont_op.cont);
213213
}
214214

215215
op.accepted_fd = accepted;

0 commit comments

Comments
 (0)