Skip to content

Commit 698db06

Browse files
authored
Immediate attachment (#794)
1 parent 7946e6a commit 698db06

28 files changed

Lines changed: 1759 additions & 39 deletions

docs/fileshare.md

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@ The method `TimeToKeep.Default` provides a recommended default for for attachmen
108108
| `AddStream` | Large payloads or data generated incrementally (recommended for large data) | Streams via `System.IO.Pipelines` with backpressure. Memory stays bounded regardless of payload size. |
109109
| `Add(Stream)` | An existing `Stream` instance is available | Bridges to `AddStream` internally via `CopyToAsync`. |
110110
| `AddFromIncoming` | The outgoing data is produced by transforming an incoming attachment of the current message | Reads from incoming and writes to the outgoing pipe at the same time. No intermediate buffer (unless `bufferSource`/`bufferSink` is set). |
111+
| `OpenOutgoingAttachment` | The handler needs the converter's output values (e.g. a "truncated" flag) before composing the outgoing message | Streams straight to storage during handler execution. No intermediate buffer. |
111112
| `AddBytes` / `AddString` | Small payloads already in memory (config, metadata, small documents) | Full payload allocated in memory. |
112113
| `Add(AttachmentFactory)` | Number of attachments not known at compile time | Dynamic. Each attachment uses the memory model of its content. |
113114
| `AddFile` | File on disk | Convenience wrapper over `AddStream`. |
@@ -269,7 +270,46 @@ class HandlerFromIncoming :
269270

270271
`bufferSink: true` runs the transform against a seekable `MemoryStream` and drains it to storage afterwards — use when the transform requires seek operations on its output (e.g. some Aspose libraries).
271272

272-
NOTE: The transform runs *during* the outgoing pipeline (after `context.Reply` / `context.Send` is called). Any value the transform produces (e.g. a "truncated" flag, encoding metadata) cannot influence the outgoing message body, since the body has already been finalized by the caller. Use eager conversion if the handler needs such values in the outgoing message.
273+
NOTE: The transform runs *during* the outgoing pipeline (after `context.Reply` / `context.Send` is called). Any value the transform produces (e.g. a "truncated" flag, encoding metadata) cannot influence the outgoing message body, since the body has already been finalized by the caller. Use `OpenOutgoingAttachment` (below) when the handler needs such values in the outgoing message.
274+
275+
276+
#### Write an outgoing attachment immediately
277+
278+
Use `OpenOutgoingAttachment` when the converter's output values (a "truncated" flag, encoding metadata, page count) need to be in the outgoing message body. The handler writes directly to storage *during* its execution and is free to read any state from the conversion before composing the reply. The library allocates the outgoing message id, sets it on the options, and registers the saved attachment so the outgoing pipeline emits the `Attachments` header but skips the deferred save.
279+
280+
<!-- snippet: OpenOutgoingAttachment -->
281+
<a id='snippet-OpenOutgoingAttachment'></a>
282+
```cs
283+
class HandlerImmediateWrite :
284+
IHandleMessages<MyMessage>
285+
{
286+
public async Task Handle(MyMessage message, HandlerContext context)
287+
{
288+
var replyOptions = new ReplyOptions();
289+
bool truncated;
290+
291+
await using (var sink = await context.OpenOutgoingAttachment(
292+
replyOptions,
293+
"TheAttachmentName"))
294+
{
295+
// Convert directly to the sink. The result (e.g. truncated)
296+
// is available before the reply body is composed.
297+
truncated = MyConverter.Convert(message.Source, sink);
298+
}
299+
300+
await context.Reply(
301+
new OtherMessage
302+
{
303+
Truncated = truncated
304+
},
305+
replyOptions);
306+
}
307+
}
308+
```
309+
<sup><a href='/src/Attachments.Sql.Tests/Snippets/Outgoing.cs#L111-L139' title='Snippet source file'>snippet source</a> | <a href='#snippet-OpenOutgoingAttachment' title='Start of snippet'>anchor</a></sup>
310+
<!-- endSnippet -->
311+
312+
NOTE: If the handler succeeds in saving but the outgoing dispatch fails (or the handler exits without ever sending the message), the saved attachment becomes an orphan. The cleanup task removes orphans by `Expiry`, but the asymmetry depends on `TransportTransactionMode`. Under `SendsAtomicWithReceive` everything is atomic; under `ReceiveOnly` / `None` the asymmetry is the same one the deferred APIs already have.
273313

274314

275315
### Reading attachments for an incoming message

docs/mdsource/attachments.include.md

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ The method `TimeToKeep.Default` provides a recommended default for for attachmen
4343
| `AddStream` | Large payloads or data generated incrementally (recommended for large data) | Streams via `System.IO.Pipelines` with backpressure. Memory stays bounded regardless of payload size. |
4444
| `Add(Stream)` | An existing `Stream` instance is available | Bridges to `AddStream` internally via `CopyToAsync`. |
4545
| `AddFromIncoming` | The outgoing data is produced by transforming an incoming attachment of the current message | Reads from incoming and writes to the outgoing pipe at the same time. No intermediate buffer (unless `bufferSource`/`bufferSink` is set). |
46+
| `OpenOutgoingAttachment` | The handler needs the converter's output values (e.g. a "truncated" flag) before composing the outgoing message | Streams straight to storage during handler execution. No intermediate buffer. |
4647
| `AddBytes` / `AddString` | Small payloads already in memory (config, metadata, small documents) | Full payload allocated in memory. |
4748
| `Add(AttachmentFactory)` | Number of attachments not known at compile time | Dynamic. Each attachment uses the memory model of its content. |
4849
| `AddFile` | File on disk | Convenience wrapper over `AddStream`. |
@@ -96,7 +97,16 @@ snippet: OutgoingFromIncoming
9697

9798
`bufferSink: true` runs the transform against a seekable `MemoryStream` and drains it to storage afterwards — use when the transform requires seek operations on its output (e.g. some Aspose libraries).
9899

99-
NOTE: The transform runs *during* the outgoing pipeline (after `context.Reply` / `context.Send` is called). Any value the transform produces (e.g. a "truncated" flag, encoding metadata) cannot influence the outgoing message body, since the body has already been finalized by the caller. Use eager conversion if the handler needs such values in the outgoing message.
100+
NOTE: The transform runs *during* the outgoing pipeline (after `context.Reply` / `context.Send` is called). Any value the transform produces (e.g. a "truncated" flag, encoding metadata) cannot influence the outgoing message body, since the body has already been finalized by the caller. Use `OpenOutgoingAttachment` (below) when the handler needs such values in the outgoing message.
101+
102+
103+
#### Write an outgoing attachment immediately
104+
105+
Use `OpenOutgoingAttachment` when the converter's output values (a "truncated" flag, encoding metadata, page count) need to be in the outgoing message body. The handler writes directly to storage *during* its execution and is free to read any state from the conversion before composing the reply. The library allocates the outgoing message id, sets it on the options, and registers the saved attachment so the outgoing pipeline emits the `Attachments` header but skips the deferred save.
106+
107+
snippet: OpenOutgoingAttachment
108+
109+
NOTE: If the handler succeeds in saving but the outgoing dispatch fails (or the handler exits without ever sending the message), the saved attachment becomes an orphan. The cleanup task removes orphans by `Expiry`, but the asymmetry depends on `TransportTransactionMode`. Under `SendsAtomicWithReceive` everything is atomic; under `ReceiveOnly` / `None` the asymmetry is the same one the deferred APIs already have.
100110

101111

102112
### Reading attachments for an incoming message

docs/sql.md

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,7 @@ The method `TimeToKeep.Default` provides a recommended default for for attachmen
253253
| `AddStream` | Large payloads or data generated incrementally (recommended for large data) | Streams via `System.IO.Pipelines` with backpressure. Memory stays bounded regardless of payload size. |
254254
| `Add(Stream)` | An existing `Stream` instance is available | Bridges to `AddStream` internally via `CopyToAsync`. |
255255
| `AddFromIncoming` | The outgoing data is produced by transforming an incoming attachment of the current message | Reads from incoming and writes to the outgoing pipe at the same time. No intermediate buffer (unless `bufferSource`/`bufferSink` is set). |
256+
| `OpenOutgoingAttachment` | The handler needs the converter's output values (e.g. a "truncated" flag) before composing the outgoing message | Streams straight to storage during handler execution. No intermediate buffer. |
256257
| `AddBytes` / `AddString` | Small payloads already in memory (config, metadata, small documents) | Full payload allocated in memory. |
257258
| `Add(AttachmentFactory)` | Number of attachments not known at compile time | Dynamic. Each attachment uses the memory model of its content. |
258259
| `AddFile` | File on disk | Convenience wrapper over `AddStream`. |
@@ -414,7 +415,46 @@ class HandlerFromIncoming :
414415

415416
`bufferSink: true` runs the transform against a seekable `MemoryStream` and drains it to storage afterwards — use when the transform requires seek operations on its output (e.g. some Aspose libraries).
416417

417-
NOTE: The transform runs *during* the outgoing pipeline (after `context.Reply` / `context.Send` is called). Any value the transform produces (e.g. a "truncated" flag, encoding metadata) cannot influence the outgoing message body, since the body has already been finalized by the caller. Use eager conversion if the handler needs such values in the outgoing message.
418+
NOTE: The transform runs *during* the outgoing pipeline (after `context.Reply` / `context.Send` is called). Any value the transform produces (e.g. a "truncated" flag, encoding metadata) cannot influence the outgoing message body, since the body has already been finalized by the caller. Use `OpenOutgoingAttachment` (below) when the handler needs such values in the outgoing message.
419+
420+
421+
#### Write an outgoing attachment immediately
422+
423+
Use `OpenOutgoingAttachment` when the converter's output values (a "truncated" flag, encoding metadata, page count) need to be in the outgoing message body. The handler writes directly to storage *during* its execution and is free to read any state from the conversion before composing the reply. The library allocates the outgoing message id, sets it on the options, and registers the saved attachment so the outgoing pipeline emits the `Attachments` header but skips the deferred save.
424+
425+
<!-- snippet: OpenOutgoingAttachment -->
426+
<a id='snippet-OpenOutgoingAttachment'></a>
427+
```cs
428+
class HandlerImmediateWrite :
429+
IHandleMessages<MyMessage>
430+
{
431+
public async Task Handle(MyMessage message, HandlerContext context)
432+
{
433+
var replyOptions = new ReplyOptions();
434+
bool truncated;
435+
436+
await using (var sink = await context.OpenOutgoingAttachment(
437+
replyOptions,
438+
"TheAttachmentName"))
439+
{
440+
// Convert directly to the sink. The result (e.g. truncated)
441+
// is available before the reply body is composed.
442+
truncated = MyConverter.Convert(message.Source, sink);
443+
}
444+
445+
await context.Reply(
446+
new OtherMessage
447+
{
448+
Truncated = truncated
449+
},
450+
replyOptions);
451+
}
452+
}
453+
```
454+
<sup><a href='/src/Attachments.Sql.Tests/Snippets/Outgoing.cs#L111-L139' title='Snippet source file'>snippet source</a> | <a href='#snippet-OpenOutgoingAttachment' title='Start of snippet'>anchor</a></sup>
455+
<!-- endSnippet -->
456+
457+
NOTE: If the handler succeeds in saving but the outgoing dispatch fails (or the handler exits without ever sending the message), the saved attachment becomes an orphan. The cleanup task removes orphans by `Expiry`, but the asymmetry depends on `TransportTransactionMode`. Under `SendsAtomicWithReceive` everything is atomic; under `ReceiveOnly` / `None` the asymmetry is the same one the deferred APIs already have.
418458

419459

420460
### Reading attachments for an incoming message
Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
[NotInParallel]
2+
public class AddFromIncomingExtraTests :
3+
IDisposable
4+
{
5+
static readonly TestState state = new();
6+
7+
[Test]
8+
public async Task BufferSinkLetsTransformSeekTheOutput()
9+
{
10+
await Run<BufferSinkMessage>("BufferSink", "abcd");
11+
await Assert.That(Encoding.UTF8.GetString(state.Bytes!)).IsEqualTo("abcd|len=4");
12+
}
13+
14+
[Test]
15+
public async Task MetadataIsAttachedToProducedOutgoingAttachment()
16+
{
17+
await Run<MetadataMessage>("Metadata", "ignored");
18+
await Assert.That(state.Metadata).IsNotNull();
19+
await Assert.That(state.Metadata!["k1"]).IsEqualTo("v1");
20+
await Assert.That(state.Metadata["k2"]).IsEqualTo("v2");
21+
}
22+
23+
[Test]
24+
public async Task TransformExceptionSurfacesAndPreventsReply()
25+
{
26+
await Run<TransformThrowsMessage>("TransformThrows", "ignored", expectReply: false);
27+
await Assert.That(state.HandlerException).IsNotNull();
28+
await Assert.That(state.HandlerException!.Message).Contains("transform boom");
29+
}
30+
31+
[Test]
32+
public async Task CleanupCallbackIsInvokedAfterCommit()
33+
{
34+
await Run<CleanupMessage>("Cleanup", "ignored");
35+
await Assert.That(state.CleanupInvoked).IsTrue();
36+
}
37+
38+
public void Dispose()
39+
{
40+
}
41+
42+
class TestState
43+
{
44+
public byte[]? Bytes;
45+
public IReadOnlyDictionary<string, string>? Metadata;
46+
public Exception? HandlerException;
47+
public bool CleanupInvoked;
48+
public ManualResetEvent Reply = new(false);
49+
}
50+
51+
static async Task Run<TMessage>(string suffix, string sourceContent, bool expectReply = true)
52+
where TMessage : IMessage, new()
53+
{
54+
state.Bytes = null;
55+
state.Metadata = null;
56+
state.HandlerException = null;
57+
state.CleanupInvoked = false;
58+
state.Reply.Reset();
59+
60+
var attachmentsPath = Path.GetFullPath($"attachments/AddFromIncomingExtra_{suffix}");
61+
if (Directory.Exists(attachmentsPath))
62+
{
63+
Directory.Delete(attachmentsPath, recursive: true);
64+
}
65+
var transportPath = Path.GetFullPath($".learningtransport/AddFromIncomingExtra_{suffix}");
66+
if (Directory.Exists(transportPath))
67+
{
68+
Directory.Delete(transportPath, recursive: true);
69+
}
70+
71+
var configuration = new EndpointConfiguration($"FileShareAddFromIncomingExtra_{suffix}");
72+
configuration.UsePersistence<LearningPersistence>();
73+
var transport = configuration.UseTransport<LearningTransport>();
74+
transport.StorageDirectory(transportPath);
75+
configuration.RegisterComponents(_ => _.AddSingleton(state));
76+
configuration.EnableAttachments(attachmentsPath, TimeToKeep.Default);
77+
configuration.UseSerialization<SystemJsonSerializer>();
78+
configuration.DisableRetries();
79+
var endpoint = await Endpoint.Start(configuration);
80+
81+
var sendOptions = new SendOptions();
82+
sendOptions.RouteToThisEndpoint();
83+
var attachments = sendOptions.Attachments();
84+
attachments.AddStream("input", async stream =>
85+
{
86+
await using var writer = new StreamWriter(stream, leaveOpen: true);
87+
await writer.WriteAsync(sourceContent);
88+
});
89+
await endpoint.Send(new TMessage(), sendOptions);
90+
91+
var fired = state.Reply.WaitOne(TimeSpan.FromSeconds(expectReply ? 10 : 5));
92+
await endpoint.Stop();
93+
94+
if (expectReply && !fired)
95+
{
96+
throw new("TimedOut waiting for reply");
97+
}
98+
}
99+
100+
class BufferSinkMessage :
101+
IMessage;
102+
103+
class MetadataMessage :
104+
IMessage;
105+
106+
class TransformThrowsMessage :
107+
IMessage;
108+
109+
class CleanupMessage :
110+
IMessage;
111+
112+
class OutMessage :
113+
IMessage;
114+
115+
class BufferSinkHandler :
116+
IHandleMessages<BufferSinkMessage>
117+
{
118+
public Task Handle(BufferSinkMessage message, HandlerContext context)
119+
{
120+
var replyOptions = new ReplyOptions();
121+
var outgoing = replyOptions.Attachments();
122+
outgoing.AddFromIncoming(
123+
fromName: "input",
124+
toName: "output",
125+
bufferSink: true,
126+
transform: async (source, sink, cancel) =>
127+
{
128+
if (!sink.CanSeek)
129+
{
130+
throw new("bufferSink: true expected the sink to be seekable.");
131+
}
132+
133+
await source.CopyToAsync(sink, cancel);
134+
// Reading sink.Length is only valid when bufferSink: true (a MemoryStream).
135+
var trailer = Encoding.UTF8.GetBytes($"|len={sink.Length}");
136+
await sink.WriteAsync(trailer, cancel);
137+
});
138+
return context.Reply(new OutMessage(), replyOptions);
139+
}
140+
}
141+
142+
class MetadataHandler :
143+
IHandleMessages<MetadataMessage>
144+
{
145+
public Task Handle(MetadataMessage message, HandlerContext context)
146+
{
147+
var replyOptions = new ReplyOptions();
148+
var outgoing = replyOptions.Attachments();
149+
outgoing.AddFromIncoming(
150+
fromName: "input",
151+
toName: "output",
152+
metadata: new Dictionary<string, string>
153+
{
154+
{"k1", "v1"},
155+
{"k2", "v2"}
156+
},
157+
transform: (source, sink, cancel) => source.CopyToAsync(sink, cancel));
158+
return context.Reply(new OutMessage(), replyOptions);
159+
}
160+
}
161+
162+
class TransformThrowsHandler :
163+
IHandleMessages<TransformThrowsMessage>
164+
{
165+
public async Task Handle(TransformThrowsMessage message, HandlerContext context)
166+
{
167+
try
168+
{
169+
var replyOptions = new ReplyOptions();
170+
var outgoing = replyOptions.Attachments();
171+
outgoing.AddFromIncoming(
172+
fromName: "input",
173+
toName: "output",
174+
transform: (_, _, _) => throw new InvalidOperationException("transform boom"));
175+
await context.Reply(new OutMessage(), replyOptions);
176+
}
177+
catch (Exception ex)
178+
{
179+
state.HandlerException = ex;
180+
state.Reply.Set();
181+
}
182+
}
183+
}
184+
185+
class CleanupHandler :
186+
IHandleMessages<CleanupMessage>
187+
{
188+
public Task Handle(CleanupMessage message, HandlerContext context)
189+
{
190+
var replyOptions = new ReplyOptions();
191+
var outgoing = replyOptions.Attachments();
192+
outgoing.AddFromIncoming(
193+
fromName: "input",
194+
toName: "output",
195+
cleanup: () => state.CleanupInvoked = true,
196+
transform: (source, sink, cancel) => source.CopyToAsync(sink, cancel));
197+
return context.Reply(new OutMessage(), replyOptions);
198+
}
199+
}
200+
201+
class OutHandler :
202+
IHandleMessages<OutMessage>
203+
{
204+
public async Task Handle(OutMessage message, HandlerContext context)
205+
{
206+
await using var memoryStream = new MemoryStream();
207+
var incoming = context.Attachments();
208+
await incoming.CopyTo("output", memoryStream, context.CancellationToken);
209+
state.Bytes = memoryStream.ToArray();
210+
await foreach (var info in incoming.GetMetadata(context.CancellationToken))
211+
{
212+
if (info.Name == "output")
213+
{
214+
state.Metadata = info.Metadata;
215+
}
216+
}
217+
state.Reply.Set();
218+
}
219+
}
220+
}

0 commit comments

Comments
 (0)