feat(2397): address PR feedback

Author: ycanales

core/htmlhelper.py

@@ -1,7 +1,5 @@
 import re
 
-import bleach
-import markdown
 from bs4 import BeautifulSoup, Comment, Tag
 from django.http import HttpHeaders
 from django.template.loader import render_to_string
@@ -754,18 +752,6 @@ def modernize_release_notes(html_content):
     return get_body_from_html(content)
 
 
-def render_whats_new_markdown(text: str) -> str:
-    """Render the constrained bullet markdown produced by the What's New
-    LLM prompt to HTML. Returns an empty string for empty input."""
-    if not text or not text.strip():
-        return ""
-    return bleach.clean(
-        markdown.markdown(text.strip(), extensions=["extra", "sane_lists"]),
-        tags=["ul", "ol", "li", "p", "strong", "em", "code", "a"],
-        attributes={"a": ["href", "title"]},
-    )
-
-
 def is_in_no_process_libs(path: str) -> bool:
     return any(lib_slug in path for lib_slug in NO_PROCESS_LIBS)
 

docs/commands.md

@@ -268,7 +268,7 @@ When a release note is freshly stored and the `Version.whats_new` field is empty
 
 ## `generate_whats_new`
 
-**Purpose**: Generate the AI-powered "What's New" draft summary for one or more Boost releases. The summary is a short, fixed-rubric bullet list (new libraries, performance, dependencies, security & reliability, developer experience) saved on the `Version` model as `whats_new` / `whats_new_html`. Drafts are not shown on the public site until an admin sets `whats_new_approved=True` (also available as a Django admin action).
+**Purpose**: Generate the AI-powered "What's New" draft summary for one or more Boost releases. The summary is a short, fixed-rubric bullet list (new libraries, performance, dependencies, security & reliability, developer experience) saved on the `Version` model as `whats_new` (markdown bullets). The public site parses the bullets into `whats_new_items` and renders them in the release-highlights card. Drafts are not shown on the public site until an admin sets `whats_new_approved=True` (also available as a Django admin action).
 
 This command is opt-in. Auto-generation only runs as a side-effect of `import_release_notes` when a version's `whats_new` is empty. Use this command to backfill historical versions or to regenerate after editing the prompt.
 

versions/admin.py

@@ -3,6 +3,7 @@
 from django.db.models.query import QuerySet
 from django.http import HttpRequest, HttpResponseRedirect
 from django.urls import path
+from django.utils.html import format_html, format_html_join
 
 from libraries.tasks import import_new_versions_tasks
 
@@ -36,7 +37,7 @@ class VersionAdmin(admin.ModelAdmin):
     date_hierarchy = "release_date"
     inlines = [VersionFileInline]
     change_list_template = "admin/version_change_list.html"
-    readonly_fields = ["whats_new_html", "whats_new_generated_at"]
+    readonly_fields = ["whats_new_items_display", "whats_new_generated_at"]
     fieldsets = (
         (
             None,
@@ -60,20 +61,35 @@ class VersionAdmin(admin.ModelAdmin):
             {
                 "fields": (
                     "whats_new",
-                    "whats_new_html",
+                    "whats_new_items_display",
                     "whats_new_approved",
                     "whats_new_generated_at",
                 ),
                 "description": (
                     "AI-generated draft summary. Edit `whats_new` (markdown bullets) "
-                    "and re-save to refresh the rendered HTML, or use the "
-                    "'Regenerate What's New' action."
+                    "and re-save to refresh the parsed items shown below, or use the "
+                    "'Regenerate What's New' action. Only bullets matching the "
+                    "`- **Label** — text` pattern are surfaced on the public site."
                 ),
             },
         ),
     )
     actions = ["approve_whats_new", "regenerate_whats_new"]
 
+    @admin.display(description="Parsed items (rendered on the site)")
+    def whats_new_items_display(self, obj: Version) -> str:
+        items = obj.whats_new_items
+        if not items:
+            return "(no parseable bullets — site will not render a What's New card)"
+        return format_html(
+            "<ul>{}</ul>",
+            format_html_join(
+                "",
+                "<li><strong>{}</strong> — {}</li>",
+                ((item["title"], item["description"]) for item in items),
+            ),
+        )
+
     def get_queryset(self, request: HttpRequest) -> QuerySet:
         # we want all versions here, including not fully_imported
         return Version.objects.with_partials()
@@ -95,25 +111,15 @@ def import_new_releases(self, request):
         self.message_user(request, msg)
         return HttpResponseRedirect("../")
 
-    def save_model(self, request, obj, form, change):
-        if change and "whats_new" in form.changed_data:
-            from core.htmlhelper import render_whats_new_markdown
-
-            obj.whats_new_html = render_whats_new_markdown(obj.whats_new)
-        super().save_model(request, obj, form, change)
-
     @admin.action(description="Approve What's New (publish)")
     def approve_whats_new(self, request, queryset):
         updated = queryset.exclude(whats_new="").update(whats_new_approved=True)
         self.message_user(request, f"Approved What's New for {updated} version(s).")
 
-    @admin.action(description="Regenerate What's New (clear + queue task)")
+    @admin.action(description="Regenerate What's New (queue task)")
     def regenerate_whats_new(self, request, queryset):
         queued = 0
         for version in queryset:
-            Version.objects.filter(pk=version.pk).update(
-                whats_new="", whats_new_html="", whats_new_approved=False
-            )
             dispatch_whats_new(version.pk)
             queued += 1
         self.message_user(request, f"Queued regeneration for {queued} version(s).")

versions/management/commands/generate_whats_new.py

@@ -15,7 +15,11 @@
     "--all-missing",
     is_flag=True,
     default=False,
-    help="Queue generation for every active version that has no summary yet.",
+    help=(
+        "Queue generation for every active version that has stored release "
+        "notes in the Rendered Content page, but no summary yet. Versions "
+        "without release notes are skipped."
+    ),
 )
 @click.option(
     "--version",
@@ -66,7 +70,7 @@ def command(
     if not all_missing and not version_slug:
         raise click.UsageError("Pass --all-missing, --version <slug>, or --validate.")
 
-    versions = _select_versions(all_missing, version_slug, force)
+    versions = _select_versions(version_slug, force)
     if not versions:
         click.secho("No versions matched.", fg="yellow")
         return
@@ -79,19 +83,15 @@ def command(
             )
             continue
         click.secho(f"queueing whats_new for {version.name}", fg="green")
-        if force:
-            Version.objects.filter(pk=version.pk).update(
-                whats_new="", whats_new_html="", whats_new_approved=False
-            )
         dispatch_whats_new(version.pk)
 
 
-def _select_versions(all_missing: bool, version_slug: str | None, force: bool):
+def _select_versions(version_slug: str | None, force: bool):
     qs = Version.objects.active().exclude(name__in=["master", "develop"])
     if version_slug:
         qs = qs.filter(slug=version_slug)
-    elif all_missing:
-        qs = qs.filter(whats_new="") if not force else qs
+    if not force:
+        qs = qs.filter(whats_new="")
 
     rendered_keys = set(
         RenderedContent.objects.filter(

versions/migrations/0027_version_whats_new.py

@@ -32,9 +32,4 @@ class Migration(migrations.Migration):
             name="whats_new_generated_at",
             field=models.DateTimeField(blank=True, null=True),
         ),
-        migrations.AddField(
-            model_name="version",
-            name="whats_new_html",
-            field=models.TextField(blank=True, default=""),
-        ),
     ]

versions/models.py

@@ -51,7 +51,6 @@ class Version(models.Model):
             "Clear to regenerate on next release-notes import."
         ),
     )
-    whats_new_html = models.TextField(blank=True, default="")
     whats_new_approved = models.BooleanField(
         default=False,
         help_text="Public site only renders the summary when this is True.",
@@ -234,10 +233,12 @@ def whats_new_items(self):
         dicts for the v3 release-highlights card.
 
         Accepts the Markdown unordered-list bullets the LLM is instructed
-        to emit; a leading ``-`` or ``*`` marker is required:
+        to emit; a leading ``-`` or ``*`` marker is required, followed by a
+        bold category label (``**Label**``):
           - `- **New libraries** — sentence`
           - `* **New libraries:** sentence`
-        Trailing `:` inside the label is stripped.
+        Trailing `:` inside the label is stripped. Bullets missing the bold
+        label are silently skipped and will not appear on the public site.
         """
         if not self.whats_new:
             return []

versions/tasks.py

@@ -626,6 +626,10 @@ def skip_tag(name, new=False):
 
 
 WHATS_NEW_MODEL = "gpt-oss-120b"
+# Guardrail to keep the LLM call comfortably under the model's context window
+# (gpt-oss-120b is ~131K tokens; ~100K chars leaves headroom for the system
+# prompt + output). Inputs above this are truncated.
+WHATS_NEW_MAX_INPUT_CHARS = 100_000
 
 WHATS_NEW_SYSTEM_PROMPT = dedent(
     """
@@ -701,6 +705,15 @@ def generate_whats_new(self, version_pk: int) -> str | None:
         return None
 
     release_note_text = _release_note_text(rendered_content)
+    if len(release_note_text) > WHATS_NEW_MAX_INPUT_CHARS:
+        logger.warning(
+            "generate_whats_new_truncating_input",
+            version_pk=version_pk,
+            original_chars=len(release_note_text),
+            max_chars=WHATS_NEW_MAX_INPUT_CHARS,
+        )
+        release_note_text = release_note_text[:WHATS_NEW_MAX_INPUT_CHARS]
+
     logger.info(
         "generate_whats_new_dispatching",
         version_pk=version_pk,
@@ -713,9 +726,15 @@ def generate_whats_new(self, version_pk: int) -> str | None:
         {"role": "user", "content": release_note_text},
     ]
     client = OpenAI(base_url=OPENROUTER_URL, api_key=OPENROUTER_API_KEY)
-    response = client.chat.completions.create(model=WHATS_NEW_MODEL, messages=messages)
     try:
+        response = client.chat.completions.create(
+            model=WHATS_NEW_MODEL, messages=messages
+        )
         content = response.choices[0].message.content
+    except OpenAIError:
+        # Let autoretry_for=(OpenAIError,) on the task handle retries.
+        logger.exception("generate_whats_new_openai_error", version_pk=version_pk)
+        raise
     except (AttributeError, IndexError) as e:
         logger.error("generate_whats_new_response_error", error=str(e))
         return None
@@ -735,15 +754,12 @@ def save_whats_new(markdown_text: str | None, version_pk: int):
     Resets ``whats_new_approved`` to False so regenerated content goes back
     through admin moderation before it becomes visible on the public site.
     """
-    from core.htmlhelper import render_whats_new_markdown
-
     if not markdown_text:
         logger.info("save_whats_new_empty_skip", version_pk=version_pk)
         return
 
     Version.objects.filter(pk=version_pk).update(
         whats_new=markdown_text,
-        whats_new_html=render_whats_new_markdown(markdown_text),
         whats_new_generated_at=timezone.now(),
         whats_new_approved=False,
     )

versions/tests/test_ai_tasks.py

@@ -2,9 +2,14 @@
 
 import pytest
 from model_bakery import baker
+from openai import APIError
 
 from core.models import RenderedContent
-from versions.tasks import generate_whats_new, save_whats_new
+from versions.tasks import (
+    WHATS_NEW_MAX_INPUT_CHARS,
+    generate_whats_new,
+    save_whats_new,
+)
 from versions.models import Version
 
 
@@ -47,7 +52,7 @@ def test_generate_whats_new_populates_field(version):
 
     version.refresh_from_db()
     assert version.whats_new == SAMPLE_OUTPUT
-    assert "<ul>" in version.whats_new_html
+    assert len(version.whats_new_items) == 3
     assert version.whats_new_generated_at is not None
     # Drafts must not auto-publish.
     assert version.whats_new_approved is False
@@ -79,22 +84,9 @@ def test_save_whats_new_skips_empty(version):
 
     version.refresh_from_db()
     assert version.whats_new == ""
-    assert version.whats_new_html == ""
     assert version.whats_new_generated_at is None
 
 
-@pytest.mark.django_db
-def test_save_whats_new_sanitizes_html(version):
-    save_whats_new.run(
-        "- ok bullet\n- <script>alert(1)</script> sneaky\n",
-        version.pk,
-    )
-
-    version.refresh_from_db()
-    assert "<script>" not in version.whats_new_html
-    assert "alert(1)" in version.whats_new_html  # text survives, tag does not
-
-
 @pytest.mark.django_db
 def test_save_whats_new_resets_approval(version):
     Version.objects.filter(pk=version.pk).update(whats_new_approved=True)
@@ -104,6 +96,47 @@ def test_save_whats_new_resets_approval(version):
     assert version.whats_new_approved is False
 
 
+@pytest.mark.django_db
+def test_generate_whats_new_truncates_long_input(version):
+    long_text = "x" * (WHATS_NEW_MAX_INPUT_CHARS + 50_000)
+    baker.make(
+        RenderedContent,
+        cache_key=version.release_notes_cache_key,
+        content_type="text/asciidoc",
+        content_original=long_text,
+    )
+
+    with patch("versions.tasks.OpenAI") as mock_openai:
+        client = mock_openai.return_value
+        client.chat.completions.create.return_value = _mock_openai_response(
+            SAMPLE_OUTPUT
+        )
+        generate_whats_new.run(version.pk)
+
+    sent_messages = client.chat.completions.create.call_args.kwargs["messages"]
+    user_content = next(m["content"] for m in sent_messages if m["role"] == "user")
+    assert len(user_content) == WHATS_NEW_MAX_INPUT_CHARS
+
+
+@pytest.mark.django_db
+def test_generate_whats_new_propagates_openai_error(version):
+    baker.make(
+        RenderedContent,
+        cache_key=version.release_notes_cache_key,
+        content_type="text/asciidoc",
+        content_original="release notes",
+    )
+
+    with patch("versions.tasks.OpenAI") as mock_openai:
+        client = mock_openai.return_value
+        # APIError is a concrete OpenAIError subclass.
+        client.chat.completions.create.side_effect = APIError(
+            message="boom", request=MagicMock(), body=None
+        )
+        with pytest.raises(APIError):
+            generate_whats_new.run(version.pk)
+
+
 @pytest.mark.django_db
 def test_whats_new_items_parses_bullets(version):
     Version.objects.filter(pk=version.pk).update(whats_new=SAMPLE_OUTPUT)

versions/tests/test_commands.py

@@ -74,9 +74,41 @@ def test_generate_whats_new_force_includes_populated(
     queued_pks = {call.args[0] for call in mock_dispatch.call_args_list}
     assert queued_pks == {version_with_notes.pk, version_with_notes_and_summary.pk}
 
-    # --force clears the existing summary so the chained save task replaces it.
+    # --force only controls which versions are queued; the existing summary
+    # is left intact until the chained save task lands its replacement.
     version_with_notes_and_summary.refresh_from_db()
-    assert version_with_notes_and_summary.whats_new == ""
+    assert version_with_notes_and_summary.whats_new != ""
+
+
+@pytest.mark.django_db
+def test_generate_whats_new_version_skips_populated_without_force(
+    version_with_notes_and_summary,
+):
+    with patch(
+        "versions.management.commands.generate_whats_new.dispatch_whats_new"
+    ) as mock_dispatch:
+        call_command(
+            "generate_whats_new", "--version", version_with_notes_and_summary.slug
+        )
+
+    mock_dispatch.assert_not_called()
+
+
+@pytest.mark.django_db
+def test_generate_whats_new_version_with_force_overrides_populated(
+    version_with_notes_and_summary,
+):
+    with patch(
+        "versions.management.commands.generate_whats_new.dispatch_whats_new"
+    ) as mock_dispatch:
+        call_command(
+            "generate_whats_new",
+            "--version",
+            version_with_notes_and_summary.slug,
+            "--force",
+        )
+
+    mock_dispatch.assert_called_once_with(version_with_notes_and_summary.pk)
 
 
 @pytest.mark.django_db