|
| 1 | +// Copyright (c) Microsoft Corporation. |
| 2 | +// Licensed under the MIT License. |
| 3 | + |
| 4 | +using Grpc.Core; |
| 5 | +using Microsoft.DurableTask.Worker.Grpc; |
| 6 | +using Microsoft.Extensions.Hosting; |
| 7 | +using Microsoft.Extensions.Logging; |
| 8 | +using Microsoft.Extensions.Logging.Abstractions; |
| 9 | +using Microsoft.Extensions.Options; |
| 10 | +using P = Microsoft.DurableTask.Protobuf; |
| 11 | + |
| 12 | +namespace Microsoft.DurableTask; |
| 13 | + |
| 14 | +/// <summary> |
| 15 | +/// Background service that drains blob-externalized payload tombstones streamed by the Durable Task |
| 16 | +/// backend and deletes the corresponding blobs from customer storage. The backend cannot delete these |
| 17 | +/// blobs itself because it has no storage credentials; it soft-deletes the payload row, pushes the token |
| 18 | +/// to this worker, and hard-deletes the row only once the worker acknowledges the blob has been removed. |
| 19 | +/// </summary> |
| 20 | +/// <remarks> |
| 21 | +/// <para> |
| 22 | +/// On the open-source worker this service is registered automatically by <c>UseExternalizedPayloads</c> |
| 23 | +/// and reuses that worker's gRPC connection. It can also be constructed directly from a |
| 24 | +/// <see cref="CallInvoker"/> (or gRPC channel) and a <see cref="PayloadStore"/> so hosts that enable large |
| 25 | +/// payloads through a different code path (for example the Durable Task Scheduler "Azure Managed" worker |
| 26 | +/// SDK) can reuse it without going through <c>UseExternalizedPayloads</c>. Because it derives from |
| 27 | +/// <see cref="BackgroundService"/> such hosts can start and stop it directly via |
| 28 | +/// <see cref="IHostedService.StartAsync"/> / <see cref="IHostedService.StopAsync"/>. |
| 29 | +/// </para> |
| 30 | +/// <para> |
| 31 | +/// It opens a single bidirectional gRPC stream and blocks awaiting server pushes when idle (a push model, |
| 32 | +/// not a polling loop) for the worker process lifetime. Deletes are idempotent, so running multiple worker |
| 33 | +/// replicas concurrently is safe. |
| 34 | +/// </para> |
| 35 | +/// </remarks> |
| 36 | +public sealed class BlobPayloadAutoPurgeService : BackgroundService |
| 37 | +{ |
| 38 | + static readonly TimeSpan ReconnectBackoffBase = TimeSpan.FromSeconds(1); |
| 39 | + static readonly TimeSpan ReconnectBackoffCap = TimeSpan.FromSeconds(30); |
| 40 | + |
| 41 | + readonly PayloadStore store; |
| 42 | + readonly ILogger logger; |
| 43 | + readonly Random jitter = new(); |
| 44 | + |
| 45 | + // Exactly one connection strategy is used. When a CallInvoker is supplied directly (cross-repo reuse) it |
| 46 | + // is used as-is; otherwise the OSS DI path resolves it lazily from the named worker options at start, |
| 47 | + // which keeps host startup resilient if the worker connection is misconfigured. |
| 48 | + readonly CallInvoker? callInvoker; |
| 49 | + readonly IOptionsMonitor<GrpcDurableTaskWorkerOptions>? workerOptions; |
| 50 | + readonly string? builderName; |
| 51 | + |
| 52 | + /// <summary> |
| 53 | + /// Initializes a new instance of the <see cref="BlobPayloadAutoPurgeService"/> class that drains the |
| 54 | + /// purge stream over the supplied gRPC call invoker. Use this constructor to reuse the service from a |
| 55 | + /// host that does not enable large payloads through <c>UseExternalizedPayloads</c>. |
| 56 | + /// </summary> |
| 57 | + /// <param name="callInvoker">The gRPC call invoker connected to the Durable Task backend.</param> |
| 58 | + /// <param name="store">The payload store used to delete blobs.</param> |
| 59 | + /// <param name="logger">The logger, or <c>null</c> to disable logging.</param> |
| 60 | + public BlobPayloadAutoPurgeService( |
| 61 | + CallInvoker callInvoker, |
| 62 | + PayloadStore store, |
| 63 | + ILogger? logger = null) |
| 64 | + : this(store, logger) |
| 65 | + { |
| 66 | + this.callInvoker = Check.NotNull(callInvoker); |
| 67 | + } |
| 68 | + |
| 69 | + /// <summary> |
| 70 | + /// Initializes a new instance of the <see cref="BlobPayloadAutoPurgeService"/> class that drains the |
| 71 | + /// purge stream over the supplied gRPC channel. Use this constructor to reuse the service from a host |
| 72 | + /// that does not enable large payloads through <c>UseExternalizedPayloads</c>. |
| 73 | + /// </summary> |
| 74 | + /// <param name="channel">The gRPC channel connected to the Durable Task backend.</param> |
| 75 | + /// <param name="store">The payload store used to delete blobs.</param> |
| 76 | + /// <param name="logger">The logger, or <c>null</c> to disable logging.</param> |
| 77 | + public BlobPayloadAutoPurgeService( |
| 78 | + ChannelBase channel, |
| 79 | + PayloadStore store, |
| 80 | + ILogger? logger = null) |
| 81 | + : this(Check.NotNull(channel).CreateCallInvoker(), store, logger) |
| 82 | + { |
| 83 | + } |
| 84 | + |
| 85 | + /// <summary> |
| 86 | + /// Initializes a new instance of the <see cref="BlobPayloadAutoPurgeService"/> class that resolves its |
| 87 | + /// gRPC connection from the named worker options. Used by the <c>UseExternalizedPayloads</c> DI path. |
| 88 | + /// </summary> |
| 89 | + /// <param name="store">The payload store used to delete blobs.</param> |
| 90 | + /// <param name="workerOptions">Monitor for the named gRPC worker options that carry the DTS connection.</param> |
| 91 | + /// <param name="builderName">The Durable Task worker builder name whose options this service uses.</param> |
| 92 | + /// <param name="logger">The logger.</param> |
| 93 | + internal BlobPayloadAutoPurgeService( |
| 94 | + PayloadStore store, |
| 95 | + IOptionsMonitor<GrpcDurableTaskWorkerOptions> workerOptions, |
| 96 | + string builderName, |
| 97 | + ILogger<BlobPayloadAutoPurgeService> logger) |
| 98 | + : this(store, logger) |
| 99 | + { |
| 100 | + this.workerOptions = Check.NotNull(workerOptions); |
| 101 | + this.builderName = Check.NotNull(builderName); |
| 102 | + } |
| 103 | + |
| 104 | + BlobPayloadAutoPurgeService(PayloadStore store, ILogger? logger) |
| 105 | + { |
| 106 | + this.store = Check.NotNull(store); |
| 107 | + this.logger = logger ?? NullLogger.Instance; |
| 108 | + } |
| 109 | + |
| 110 | + /// <inheritdoc/> |
| 111 | + protected override async Task ExecuteAsync(CancellationToken stoppingToken) |
| 112 | + { |
| 113 | + CallInvoker? callInvoker = this.ResolveCallInvoker(); |
| 114 | + if (callInvoker is null) |
| 115 | + { |
| 116 | + // ResolveCallInvoker already logged why the purge stream cannot run. |
| 117 | + return; |
| 118 | + } |
| 119 | + |
| 120 | + P.TaskHubSidecarService.TaskHubSidecarServiceClient client = new(callInvoker); |
| 121 | + TimeSpan backoff = ReconnectBackoffBase; |
| 122 | + |
| 123 | + while (!stoppingToken.IsCancellationRequested) |
| 124 | + { |
| 125 | + try |
| 126 | + { |
| 127 | + await this.DrainPurgeStreamAsync(client, stoppingToken); |
| 128 | + |
| 129 | + // A clean server-side completion just means "nothing more to purge right now"; reconnect promptly. |
| 130 | + backoff = ReconnectBackoffBase; |
| 131 | + } |
| 132 | + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) |
| 133 | + { |
| 134 | + break; |
| 135 | + } |
| 136 | + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) |
| 137 | + { |
| 138 | + Logs.AutoPurgeStreamError(this.logger, ex); |
| 139 | + await this.DelayWithJitterAsync(backoff, stoppingToken); |
| 140 | + backoff = NextBackoff(backoff); |
| 141 | + } |
| 142 | + } |
| 143 | + } |
| 144 | + |
| 145 | + static TimeSpan NextBackoff(TimeSpan current) |
| 146 | + { |
| 147 | + if (current >= ReconnectBackoffCap) |
| 148 | + { |
| 149 | + return ReconnectBackoffCap; |
| 150 | + } |
| 151 | + |
| 152 | + long doubledTicks = current.Ticks * 2; |
| 153 | + return doubledTicks >= ReconnectBackoffCap.Ticks ? ReconnectBackoffCap : TimeSpan.FromTicks(doubledTicks); |
| 154 | + } |
| 155 | + |
| 156 | + CallInvoker? ResolveCallInvoker() |
| 157 | + { |
| 158 | + // Cross-repo reuse path: the caller supplied a ready-to-use connection. |
| 159 | + if (this.callInvoker is not null) |
| 160 | + { |
| 161 | + return this.callInvoker; |
| 162 | + } |
| 163 | + |
| 164 | + // OSS DI path: reuse the same connection the worker uses to reach DTS. After UseExternalizedPayloads |
| 165 | + // runs, the named options are guaranteed to expose a CallInvoker (a Channel, if supplied, is folded |
| 166 | + // into one during PostConfigure). |
| 167 | + CallInvoker? resolved; |
| 168 | + try |
| 169 | + { |
| 170 | + resolved = this.workerOptions?.Get(this.builderName!).CallInvoker; |
| 171 | + } |
| 172 | + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) |
| 173 | + { |
| 174 | + Logs.AutoPurgeConfigurationError(this.logger, ex); |
| 175 | + return null; |
| 176 | + } |
| 177 | + |
| 178 | + if (resolved is null) |
| 179 | + { |
| 180 | + Logs.AutoPurgeNoCallInvoker(this.logger); |
| 181 | + } |
| 182 | + |
| 183 | + return resolved; |
| 184 | + } |
| 185 | + |
| 186 | + async Task DrainPurgeStreamAsync( |
| 187 | + P.TaskHubSidecarService.TaskHubSidecarServiceClient client, CancellationToken stoppingToken) |
| 188 | + { |
| 189 | + using AsyncDuplexStreamingCall<P.PayloadPurged, P.TombstonedPayload> call = |
| 190 | + client.PurgeExternalPayloads(cancellationToken: stoppingToken); |
| 191 | + |
| 192 | + Logs.AutoPurgeStreamOpened(this.logger); |
| 193 | + |
| 194 | + while (await call.ResponseStream.MoveNext(stoppingToken)) |
| 195 | + { |
| 196 | + P.TombstonedPayload tombstone = call.ResponseStream.Current; |
| 197 | + try |
| 198 | + { |
| 199 | + await this.store.DeleteAsync(tombstone.Token, stoppingToken); |
| 200 | + } |
| 201 | + catch (Exception ex) when ( |
| 202 | + ex is not OperationCanceledException and not OutOfMemoryException and not StackOverflowException) |
| 203 | + { |
| 204 | + // Leave the payload soft-deleted so the backend can re-push it later; skip the ack. |
| 205 | + Logs.AutoPurgeDeleteFailed(this.logger, ex, tombstone.Token); |
| 206 | + continue; |
| 207 | + } |
| 208 | + |
| 209 | + // The ack write inherits cancellation from the duplex call (opened with stoppingToken); the |
| 210 | + // WriteAsync(message, CancellationToken) overload is unavailable on netstandard2.0. |
| 211 | +#pragma warning disable CA2016 // Forward the CancellationToken parameter to methods that take one |
| 212 | + await call.RequestStream.WriteAsync(new P.PayloadPurged |
| 213 | + { |
| 214 | + PartitionId = tombstone.PartitionId, |
| 215 | + InstanceKey = tombstone.InstanceKey, |
| 216 | + PayloadId = tombstone.PayloadId, |
| 217 | + }); |
| 218 | +#pragma warning restore CA2016 |
| 219 | + } |
| 220 | + |
| 221 | + await call.RequestStream.CompleteAsync(); |
| 222 | + } |
| 223 | + |
| 224 | + async Task DelayWithJitterAsync(TimeSpan maxDelay, CancellationToken cancellationToken) |
| 225 | + { |
| 226 | + // Full jitter in [0, maxDelay) spreads reconnect attempts across replicas. |
| 227 | + TimeSpan delay = TimeSpan.FromTicks((long)(maxDelay.Ticks * this.jitter.NextDouble())); |
| 228 | + if (delay > TimeSpan.Zero) |
| 229 | + { |
| 230 | + await Task.Delay(delay, cancellationToken); |
| 231 | + } |
| 232 | + } |
| 233 | +} |
0 commit comments