Fix newsletter trigger semantics, Monday roundup gating, roundup idempotency, and website-order section rendering

src/TechHub.Api/Endpoints/NewsletterEndpoints.cs

@@ -279,10 +279,13 @@ private static async Task<IResult> GetSendLogAsync(
         return Results.Ok(log);
     }
 
-    private static IResult TriggerNewsletterAsync(NewsletterBackgroundService backgroundService)
+    private static IResult TriggerNewsletterAsync(
+        [FromQuery] string? kind,
+        NewsletterBackgroundService backgroundService)
     {
-        backgroundService.TriggerImmediateRun();
-        return Results.Accepted("/api/admin/newsletter/send-log", new { message = "Newsletter send triggered" });
+        var normalizedKind = string.Equals(kind?.Trim(), "daily", StringComparison.OrdinalIgnoreCase) ? "daily" : "roundup";
+        backgroundService.TriggerImmediateRun(normalizedKind);
+        return Results.Accepted("/api/admin/newsletter/send-log", new { message = $"Newsletter {normalizedKind} send triggered" });
     }
 
     private static IResult TestSendNewsletterAsync(

src/TechHub.Api/Services/NewsletterBackgroundService.cs

@@ -1,4 +1,5 @@
 using System.Data;
+using System.Diagnostics.CodeAnalysis;
 using Dapper;
 using Microsoft.Extensions.Options;
 using TechHub.Core.Configuration;
@@ -17,6 +18,7 @@ public sealed class NewsletterBackgroundService : BackgroundService
     private string? _pendingTestSendEmail;
     private IReadOnlyList<string>? _pendingTestSections;
     private string? _pendingTestSendKind;
+    private string? _pendingManualKind;
 
     public NewsletterBackgroundService(
         IServiceProvider serviceProvider,
@@ -35,11 +37,12 @@ public NewsletterBackgroundService(
         _logger = logger;
     }
 
-    public void TriggerImmediateRun()
+    public void TriggerImmediateRun(string kind = "roundup")
     {
         _pendingTestSendEmail = null;
         _pendingTestSections = null;
         _pendingTestSendKind = null;
+        _pendingManualKind = kind;
         _manualTrigger.TrySetResult(true);
     }
 
@@ -104,6 +107,15 @@ private async Task RunManualAsync(CancellationToken ct)
             return;
         }
 
+        var manualKind = string.Equals(_pendingManualKind?.Trim(), "daily", StringComparison.OrdinalIgnoreCase) ? "daily" : "roundup";
+        _pendingManualKind = null;
+
+        if (string.Equals(manualKind, "daily", StringComparison.OrdinalIgnoreCase))
+        {
+            await SendDailyEmailsAsync(enforceHourGate: false, ct);
+            return;
+        }
+
         await SendLatestRoundupsAsync(ct);
     }
 
@@ -115,26 +127,51 @@ private async Task SendLatestRoundupsAsync(CancellationToken ct)
 
         const string Sql = """
             SELECT DISTINCT ON (primary_section_name)
-                slug
+                slug AS Slug,
+                date_epoch AS DateEpoch
             FROM content_items
             WHERE collection_name = 'roundups'
               AND primary_section_name IS NOT NULL
               AND primary_section_name <> 'all'
             ORDER BY primary_section_name, date_epoch DESC
             """;
 
-        var slugs = (await connection.QueryAsync<string>(new CommandDefinition(Sql, cancellationToken: ct))).AsList();
+        var latestRoundups = (await connection.QueryAsync<LatestSectionRoundupRow>(new CommandDefinition(Sql, cancellationToken: ct))).AsList();
+        if (latestRoundups.Count == 0)
+        {
+            return;
+        }
+
+        var roundupTimeZone = ResolveRoundupTimeZone();
+        var expectedMonday = GetExpectedRoundupMonday(DateTimeOffset.UtcNow, roundupTimeZone);
+        if (!latestRoundups.All(r => GetRoundupDate(r.DateEpoch, roundupTimeZone) == expectedMonday))
+        {
+            _logger.LogInformation(
+                "Skipping roundup newsletter send because latest section roundups do not match expected Monday {ExpectedMonday}",
+                expectedMonday);
+            return;
+        }
+
+        var slugs = latestRoundups.Select(r => r.Slug).ToList();
         if (slugs.Count > 0)
         {
-            await newsletterService.SendCombinedWeeklyAsync(slugs, ct);
+            await newsletterService.SendCombinedWeeklyAsync(
+                slugs,
+                sendTargetKey: expectedMonday.ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture),
+                ct);
         }
     }
 
     private async Task SendScheduledDailyEmailsAsync(CancellationToken ct)
+    {
+        await SendDailyEmailsAsync(enforceHourGate: true, ct);
+    }
+
+    private async Task SendDailyEmailsAsync(bool enforceHourGate, CancellationToken ct)
     {
         var timeZone = ResolveTimeZone(_options.DailyDigestTimeZoneId);
         var localNow = TimeZoneInfo.ConvertTime(DateTimeOffset.UtcNow, timeZone);
-        if (localNow.Hour != _options.DailyDigestHourLocal)
+        if (enforceHourGate && localNow.Hour != _options.DailyDigestHourLocal)
         {
             return;
         }
@@ -146,6 +183,9 @@ private async Task SendScheduledDailyEmailsAsync(CancellationToken ct)
         await newsletterService.SendAdminStatusReportAsync(day, ct);
     }
 
+    [SuppressMessage("Performance", "CA1812", Justification = "Instantiated by Dapper materialization.")]
+    private sealed record LatestSectionRoundupRow(string Slug, long DateEpoch);
+
     private async Task<bool> IsEnabledAsync(CancellationToken ct)
     {
         // If disabled via environment variable (Newsletter__ScheduledSendEnabled=false), skip the
@@ -215,4 +255,22 @@ private TimeZoneInfo ResolveTimeZone(string configuredId)
 
         return TimeZoneInfo.Utc;
     }
+
+    private static TimeZoneInfo ResolveRoundupTimeZone()
+    {
+        return TimeZoneInfo.FindSystemTimeZoneById(
+            OperatingSystem.IsWindows() ? "Romance Standard Time" : "Europe/Brussels");
+    }
+
+    private static DateOnly GetExpectedRoundupMonday(DateTimeOffset utcNow, TimeZoneInfo roundupTimeZone)
+    {
+        var localDate = DateOnly.FromDateTime(TimeZoneInfo.ConvertTime(utcNow, roundupTimeZone).Date);
+        var daysSinceMonday = ((int)localDate.DayOfWeek - (int)DayOfWeek.Monday + 7) % 7;
+        return localDate.AddDays(-daysSinceMonday);
+    }
+
+    private static DateOnly GetRoundupDate(long dateEpoch, TimeZoneInfo roundupTimeZone)
+    {
+        return DateOnly.FromDateTime(TimeZoneInfo.ConvertTime(DateTimeOffset.FromUnixTimeSeconds(dateEpoch), roundupTimeZone).Date);
+    }
 }

src/TechHub.Core/Interfaces/INewsletterService.cs

@@ -5,7 +5,7 @@ namespace TechHub.Core.Interfaces;
 public interface INewsletterService
 {
     Task<bool> SendRoundupNewsletterAsync(string roundupSlug, CancellationToken ct = default);
-    Task<bool> SendCombinedWeeklyAsync(IReadOnlyList<string> roundupSlugs, CancellationToken ct = default);
+    Task<bool> SendCombinedWeeklyAsync(IReadOnlyList<string> roundupSlugs, string? sendTargetKey = null, CancellationToken ct = default);
     Task<bool> SendTestEmailAsync(string roundupSlug, string recipientEmail, CancellationToken ct = default);
     Task<bool> SendTestWeeklyAsync(IReadOnlyList<string> sections, string recipientEmail, CancellationToken ct = default);
     Task<bool> SendTestDailyEmailAsync(IReadOnlyList<string> sections, string recipientEmail, CancellationToken ct = default);

src/TechHub.Infrastructure/Services/Newsletter/NewsletterService.cs

@@ -153,52 +153,42 @@ await _subscriberRepository.LogSendAsync(
         return sent;
     }
 
-    public async Task<bool> SendCombinedWeeklyAsync(IReadOnlyList<string> roundupSlugs, CancellationToken ct = default)
+    public async Task<bool> SendCombinedWeeklyAsync(IReadOnlyList<string> roundupSlugs, string? sendTargetKey = null, CancellationToken ct = default)
     {
         ArgumentNullException.ThrowIfNull(roundupSlugs);
         if (roundupSlugs.Count == 0)
         {
             return false;
         }
 
-        if (!IsUnsubscribeSecretConfigured("combined-weekly"))
-        {
-            foreach (var slug in roundupSlugs)
-            {
-                await _subscriberRepository.LogSendAsync("weekly-roundup", slug, 0, "failed", "Unsubscribe secret is not configured", ct);
-            }
-
-            return false;
-        }
-
-        // Skip slugs that have already been sent
-        var newSlugs = new List<string>(roundupSlugs.Count);
+        // Load roundup content for each slug
+        var roundups = new List<RoundupRow>(roundupSlugs.Count);
         foreach (var slug in roundupSlugs)
         {
-            if (!await _subscriberRepository.HasBeenSentAsync("weekly-roundup", slug, ct))
+            var roundup = await GetRoundupBySlugAsync(slug, ct);
+            if (roundup is not null)
             {
-                newSlugs.Add(slug);
+                roundups.Add(roundup);
             }
         }
 
-        if (newSlugs.Count == 0)
+        if (roundups.Count == 0)
         {
             return false;
         }
 
-        // Load roundup content for each new slug
-        var roundups = new List<RoundupRow>(newSlugs.Count);
-        foreach (var slug in newSlugs)
+        var targetKey = string.IsNullOrWhiteSpace(sendTargetKey)
+            ? $"batch:{string.Join(",", roundups.Select(r => r.Slug).OrderBy(s => s, StringComparer.OrdinalIgnoreCase))}"
+            : sendTargetKey.Trim();
+
+        if (await _subscriberRepository.HasBeenSentAsync("weekly-roundup", targetKey, ct))
         {
-            var roundup = await GetRoundupBySlugAsync(slug, ct);
-            if (roundup is not null)
-            {
-                roundups.Add(roundup);
-            }
+            return false;
         }
 
-        if (roundups.Count == 0)
+        if (!IsUnsubscribeSecretConfigured("combined-weekly"))
         {
+            await _subscriberRepository.LogSendAsync("weekly-roundup", targetKey, 0, "failed", "Unsubscribe secret is not configured", ct);
             return false;
         }
 
@@ -215,8 +205,8 @@ public async Task<bool> SendCombinedWeeklyAsync(IReadOnlyList<string> roundupSlu
 
         var roundupsBySection = roundups.ToDictionary(r => r.SectionName, StringComparer.OrdinalIgnoreCase);
 
-        // Track per-slug delivery counts: (expected recipients, successful sends)
-        var slugStats = newSlugs.ToDictionary(s => s, _ => (Expected: 0, Actual: 0), StringComparer.OrdinalIgnoreCase);
+        var attempted = 0;
+        var successful = 0;
 
         try
         {
@@ -227,16 +217,14 @@ public async Task<bool> SendCombinedWeeklyAsync(IReadOnlyList<string> roundupSlu
                     .Where(s => roundupsBySection.ContainsKey(s))
                     .Select(s => roundupsBySection[s])
                     .ToList();
+                relevantRoundups = OrderRoundupsByWebsiteOrder(relevantRoundups);
 
                 if (relevantRoundups.Count == 0)
                 {
                     continue;
                 }
 
-                foreach (var r in relevantRoundups)
-                {
-                    slugStats[r.Slug] = (slugStats[r.Slug].Expected + 1, slugStats[r.Slug].Actual);
-                }
+                attempted++;
 
                 var unsubscribeUrl = BuildUnsubscribeUrl(subscriber.Email);
                 var manageUrl = BuildManageUrl(subscriber.Email, _options.UnsubscribeSecret);
@@ -248,49 +236,24 @@ public async Task<bool> SendCombinedWeeklyAsync(IReadOnlyList<string> roundupSlu
 
                 if (await SendEmailAsync(subscriber.Email, subject, html, text, ct))
                 {
-                    foreach (var r in relevantRoundups)
-                    {
-                        slugStats[r.Slug] = (slugStats[r.Slug].Expected, slugStats[r.Slug].Actual + 1);
-                    }
+                    successful++;
                 }
             }
         }
         catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException)
         {
-            foreach (var slug in newSlugs)
-            {
-                var actual = slugStats[slug].Actual;
-                var status = actual > 0 ? "partial" : "failed";
-                await _subscriberRepository.LogSendAsync("weekly-roundup", slug, actual, status, ex.Message, ct);
-            }
-
+            var status = successful > 0 ? "partial" : "failed";
+            await _subscriberRepository.LogSendAsync("weekly-roundup", targetKey, successful, status, ex.Message, ct);
             _logger.LogError(ex, "Failed sending combined weekly newsletter");
             return false;
         }
 
-        // Log each slug with its individual delivery outcome
-        var anySent = false;
-        foreach (var slug in newSlugs)
-        {
-            var (expected, actual) = slugStats[slug];
-            if (expected == 0)
-            {
-                await _subscriberRepository.LogSendAsync("weekly-roundup", slug, 0, "sent", null, ct);
-                anySent = true;
-            }
-            else
-            {
-                var status = actual == expected ? "sent" : actual > 0 ? "partial" : "failed";
-                var error = actual < expected ? $"Delivered to {actual} of {expected} subscribers" : null;
-                await _subscriberRepository.LogSendAsync("weekly-roundup", slug, actual, status, error, ct);
-                if (actual > 0)
-                {
-                    anySent = true;
-                }
-            }
-        }
-
-        return anySent;
+        var sendStatus = attempted == 0 || successful == attempted ? "sent" : successful > 0 ? "partial" : "failed";
+        var sendError = attempted == 0 || successful == attempted
+            ? null
+            : $"Delivered to {successful} of {attempted} subscribers";
+        await _subscriberRepository.LogSendAsync("weekly-roundup", targetKey, successful, sendStatus, sendError, ct);
+        return attempted == 0 || successful > 0;
     }
 
     public async Task<bool> SendTestWeeklyAsync(IReadOnlyList<string> sections, string recipientEmail, CancellationToken ct = default)
@@ -340,6 +303,8 @@ LIMIT 1
             return false;
         }
 
+        roundups = OrderRoundupsByWebsiteOrder(roundups);
+
         var logKey = $"test-weekly@{DateTime.UtcNow:yyyyMMddHHmmss}";
         var unsubscribeUrl = BuildUnsubscribeUrl(recipientEmail);
         var manageUrl = BuildManageUrl(recipientEmail, _options.UnsubscribeSecret);
@@ -376,7 +341,7 @@ public async Task<bool> SendTestDailyEmailAsync(IReadOnlyList<string> sections,
 
         var yesterday = DateOnly.FromDateTime(DateTime.UtcNow.AddDays(-1));
         var (start, end) = GetDayUtcWindow(yesterday);
-        var sectionNames = sections.Select(s => s.Trim().ToLowerInvariant()).ToList();
+        var sectionNames = OrderSectionsByWebsiteOrder(sections);
         var itemsBySection = await GetDailyItemsBySectionAsync(sectionNames, start, end, ct);
         var html = BuildDailyOverviewHtml(yesterday, sectionNames, itemsBySection, recipientEmail);
         var text = BuildDailyOverviewText(yesterday, sectionNames, itemsBySection, recipientEmail);
@@ -517,6 +482,7 @@ public async Task<bool> SendDailyOverviewAsync(DateOnly day, CancellationToken c
             var selectedSections = subscriber.DailySections
                 .Where(section => itemsBySection.ContainsKey(section))
                 .ToList();
+            selectedSections = OrderSectionsByWebsiteOrder(selectedSections);
 
             var html = BuildDailyOverviewHtml(day, selectedSections, itemsBySection, subscriber.Email);
             var text = BuildDailyOverviewText(day, selectedSections, itemsBySection, subscriber.Email);
@@ -930,6 +896,37 @@ private string BuildCombinedWeeklyPlainText(IReadOnlyList<RoundupRow> roundups,
     private string GetSectionTitle(string slug) =>
         _appSettings.Content.Sections.TryGetValue(slug, out var config) ? config.Title : slug;
 
+    private List<string> OrderSectionsByWebsiteOrder(IEnumerable<string> sectionNames)
+    {
+        var sectionOrder = _appSettings.Content.Sections
+            .Where(kvp => !string.Equals(kvp.Key, "all", StringComparison.OrdinalIgnoreCase))
+            .ToDictionary(
+                kvp => kvp.Key,
+                kvp => kvp.Value.Order,
+                StringComparer.OrdinalIgnoreCase);
+
+        return sectionNames
+            .Where(s => !string.IsNullOrWhiteSpace(s))
+            .Select(s => s.Trim().ToLowerInvariant())
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .OrderBy(s => sectionOrder.TryGetValue(s, out var order) ? order : int.MaxValue)
+            .ThenBy(s => GetSectionTitle(s), StringComparer.OrdinalIgnoreCase)
+            .ToList();
+    }
+
+    private List<RoundupRow> OrderRoundupsByWebsiteOrder(IEnumerable<RoundupRow> roundups)
+    {
+        var orderedSections = OrderSectionsByWebsiteOrder(roundups.Select(r => r.SectionName));
+        var sectionIndex = orderedSections
+            .Select((section, index) => new { section, index })
+            .ToDictionary(x => x.section, x => x.index, StringComparer.OrdinalIgnoreCase);
+
+        return roundups
+            .OrderBy(r => sectionIndex.TryGetValue(r.SectionName, out var index) ? index : int.MaxValue)
+            .ThenBy(r => GetSectionTitle(r.SectionName), StringComparer.OrdinalIgnoreCase)
+            .ToList();
+    }
+
     private string BuildDailyOverviewHtml(
         DateOnly day,
         IReadOnlyList<string> sections,