Story 2096: Implement Wagtail Integration (#2100)

Co-authored-by: Teodoro B. Mendes <teodoro.mendes@metalab.com>

Author: jlchilders11

config/settings.py

@@ -104,6 +104,7 @@
     "wagtail.images",
     "wagtail.search",
     "wagtail.admin",
+    "wagtail.contrib.routable_page",
     "wagtail",
     "wagtailmarkdown",
     "modelcluster",
@@ -124,6 +125,7 @@
     "slack",
     "testimonials",
     "patches",
+    "pages",
     "asciidoctor_sandbox",
 ]
 

config/urls.py

@@ -43,7 +43,7 @@
     StaticContentTemplateView,
     UserGuideTemplateView,
 )
-from marketing.views import PlausibleRedirectView, WhitePaperView
+from marketing.views import PlausibleRedirectView
 from libraries.api import LibrarySearchView
 from libraries.views import (
     LibraryDetail,
@@ -134,11 +134,6 @@
             PlausibleRedirectView.as_view(),
             name="bsm",
         ),
-        path(
-            "outreach/<slug:category>/<slug:slug>",
-            WhitePaperView.as_view(),
-            name="whitepaper",
-        ),
         path(
             "accounts/social/signup/",
             CustomSocialSignupViewView.as_view(),
@@ -424,6 +419,10 @@
         ),
     ]
     + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+    + [
+        path("outreach/", include(wagtail_urls)),
+        path("pages/", include(wagtail_urls)),
+    ]
     + [
         # Libraries docs, some HTML parts are re-written
         re_path(
@@ -457,10 +456,6 @@
         ),
     ]
     + djdt_urls
-    + [
-        # Wagtail catch-all (must be last!)
-        path("", include(wagtail_urls)),
-    ]
 )
 
 handler404 = "ak.views.custom_404_view"

core/mixins.py

@@ -42,6 +42,11 @@ def get_context_data(self, **kwargs):
             context = super().get_context_data(**kwargs)
         return context
 
+    def serve(self, request, *args, **kwargs):
+        if not flag_is_active(request, "v3"):
+            raise Http404
+        return super().serve(request, *args, **kwargs)
+
     def get_v3_context_data(self, **kwargs):
         """Override in subclasses to provide v3-specific context."""
         return {**kwargs}

marketing/models.py

@@ -148,8 +148,15 @@ class DetailPage(EmailCapturePage):
 class OutreachHomePage(Page):
     """A dummy homepage to just return a 404 at the `/outreach/` url"""
 
-    parent_page_types = ["wagtailcore.Page"]
-    subpage_types = ["marketing.ProgramPageIndex", "marketing.TopicPage"]
+    parent_page_types = [
+        "wagtailcore.Page",
+        "pages.RoutableHomePage",
+    ]
+    subpage_types = [
+        "marketing.ProgramPageIndex",
+        "marketing.TopicPage",
+        "testimonials.TestimonialsIndexPage",
+    ]
     max_count = 1  # one container
 
     def route(self, request, path_components):
@@ -158,13 +165,14 @@ def route(self, request, path_components):
         /outreach/program_page/<slug>/ => delegate to ProgramPageIndex -> ProgramPage
         /outreach/<topic>/<detail>/ => delegate to TopicPage -> DetailPage
         """
+        print(path_components)
         if not path_components:
             return RouteResult(self)
 
-        _, second, *rest = path_components
+        first, *rest = path_components
 
         # Fixed segment for program pages
-        if second == "program_page":
+        if first == "program_page":
             try:
                 program_page_index = ProgramPageIndex.objects.child_of(self).get()
             except ProgramPageIndex.DoesNotExist:
@@ -174,7 +182,7 @@ def route(self, request, path_components):
 
         # Otherwise, first segment should be a TopicPage slug
         try:
-            topic = TopicPage.objects.child_of(self).get(slug=second)
+            topic = TopicPage.objects.child_of(self).get(slug=first)
         except TopicPage.DoesNotExist:
             raise Http404("Topic not found")
 

marketing/tests.py

@@ -4,10 +4,6 @@
 import pytest
 
 
-def test_whitepaper_view(tp):
-    tp.assertGoodView("whitepaper", slug="_example")
-
-
 @pytest.mark.parametrize("url_stem", ["qrc", "bsm"])
 def test_plausible_redirect_and_plausible_payload(tp, url_stem):
     """XFF present; querystring preserved; payload/headers correct."""

news/management/commands/backpopulate_thumbnails.py

@@ -2,7 +2,9 @@
 from celery import group
 
 from news.models import Video
-from news.tasks import set_thumbnail_for_video_entry
+from news.tasks import set_thumbnail_for_video_entry, set_thumbnail_for_video_page
+
+from pages.models import PostPage
 
 
 @click.command()
@@ -14,22 +16,27 @@
 def command(dry_run):
     """Backpopulate video entry thumbnails that have not been set"""
 
-    qs = Video.objects.filter(thumbnail__isnull=True)
-    count = qs.count()
+    v_qs = Video.objects.filter(thumbnail__isnull=True)
+    p_qs = PostPage.objects.filter(
+        video_thumbnail__isnull=True, content__0__type="video"
+    )
+    count = v_qs.count() + p_qs.count()
 
     if count == 0:
         click.echo("No entries found without thumbnails.")
         return
 
     if dry_run:
         click.echo(f"Would process {count} entries:")
-        for entry in qs[:10]:
+        for entry in (list(v_qs) + list(p_qs))[:10]:
             click.echo(f"  - {entry.pk}: {entry.title}")
         if count > 10:
             click.echo(f"  ... and {count - 10} more")
         return
 
     click.echo("Dispatching thumbnail tasks...")
-    g = group(set_thumbnail_for_video_entry.s(i.pk) for i in qs.iterator())
+    g = group(set_thumbnail_for_video_entry.s(i.pk) for i in v_qs.iterator())
     g.apply_async()
+    g_2 = group(set_thumbnail_for_video_page.s(i.pk) for i in p_qs.iterator())
+    g_2.apply_async()
     click.echo(f"Thumbnail tasks for {count} tasks queued.")

news/models.py

@@ -162,6 +162,13 @@ def video_thumbnail(self):
             result = None
         return result
 
+    @property
+    def image_url(self):
+        if not self.image:
+            return ""
+        else:
+            return self.image.url
+
     @cached_property
     def tag(self):
         return getattr(self, "_tag", self.news_type)
@@ -230,6 +237,9 @@ def approve(self, user, commit=True):
         if commit:
             self.save(update_fields=["moderator", "approved_at", "modified_at"])
 
+    def get_content(self):
+        return self.content
+
     @cached_property
     def use_summary(self):
         return self.summary and (

news/tasks.py

@@ -6,7 +6,7 @@
 from config.celery import app
 from config.settings import OPENROUTER_API_KEY, OPENROUTER_URL, SUMMARIZATION_MODEL
 from news.constants import CONTENT_SUMMARIZATION_THRESHOLD
-from news.helpers import UnsafeURLError, extract_article, safe_get
+from news.helpers import UnsafeURLError, extract_article, safe_get, extract_content
 from news.utils import set_video_thumbnail
 
 logger = structlog.get_logger(__name__)
@@ -138,18 +138,52 @@ def save_entry_summary_value(summary: str | None, pk: int):
 
 
 @app.task
-def summary_dispatcher(pk: int):
+def save_page_summary_value(summary: str, pk: int):
+    from pages.models import PostPage
+
+    # generate_summary returns None/"" on malformed or empty model output; saving
+    # that would clobber an existing Entry.summary, so treat it as "do not save".
+    if not summary:
+        logger.warning(f"Skipping summary save for {pk=}: empty/malformed model output")
+        return
+
+    page = PostPage.objects.get(pk=pk)
+    page.summary = summary
+    page.save()
+
+
+@app.task
+def summary_dispatcher(pk: int, model_name: str = "Entry"):
     from news.models import Entry
+    from pages.models import PostPage
+
+    if model_name == "PostPage":
+        model = PostPage
+    elif model_name == "Entry":
+        model = Entry
+    else:
+        logger.warning("Invalid Model Name passed to summary dispatcher")
+        return
 
-    entry = Entry.objects.get(pk=pk)
-    logger.info(f"Dispatching {pk=} with {entry.news_type=}")
-    handler = {
-        "news": set_summary_for_event_entry,
-        "blogpost": set_summary_for_event_entry,
-        "link": set_summary_for_link_entry,
-        "video": set_summary_for_video_entry,
-        "poll": set_summary_for_poll_entry,
-    }[entry.determined_news_type]
+    entry = model.objects.get(pk=pk)
+    if model == Entry:
+        logger.info(f"Dispatching {pk=} with {entry.news_type=}")
+        handler = {
+            "news": set_summary_for_event_entry,
+            "blogpost": set_summary_for_event_entry,
+            "link": set_summary_for_link_entry,
+            "video": set_summary_for_video_entry,
+            "poll": set_summary_for_poll_entry,
+        }[entry.determined_news_type.lower()]
+    elif model == PostPage:
+        logger.info(f"Dispatching {pk=} with {entry.post_content_type=}")
+        handler = {
+            "news": set_summary_for_event_page,
+            "blogpost": set_summary_for_event_page,
+            "link": set_summary_for_link_page,
+            "video": set_summary_for_video_page,
+            "poll": set_summary_for_poll_page,
+        }[entry.determined_news_type.lower()]
     logger.info(f"Dispatching summary task for {pk=} to {handler.__name__=}")
     handler.delay(pk)
 
@@ -240,3 +274,68 @@ def sync_post_views_from_plausible():
 
     updated = update_page_views(slug_views)
     logger.info("sync_post_views.done", updated=updated)
+
+
+# These tasks emulate the functionality of setting the summmary, but for PostPages instead of Entry.
+# Once the Entry model has been depricated and removed, the above functions can be deleted and the selector logic
+# removed from the dispatcher.
+
+
+@app.task
+def set_summary_for_event_page(pk: int):
+    from pages.models import PostPage
+
+    page = PostPage.objects.get(pk=pk)
+    content = page.content[0].value
+    logger.info(f"dispatching summarize task for {pk=} with {content[:40]=}...")
+    if content and len(content) < CONTENT_SUMMARIZATION_THRESHOLD:
+        logger.warning(f"Content too short to summarize for {pk=}, skipping.")
+        return
+    logger.info(f"handing off {pk=} to summarize_content task")
+    summarize_content.apply_async(
+        (content, page.title, SUMMARIZATION_MODEL),
+        link=save_page_summary_value.s(pk),
+    )
+
+
+@app.task
+def set_summary_for_link_page(pk: int):
+    logger.info(f"Setting summary for link page {pk=}")
+    from pages.models import PostPage
+
+    page = PostPage.objects.get(pk=pk)
+    external_url = page.external_url
+    try:
+        logger.info(f"Fetching content from {external_url=} for entry.{pk=}")
+        response = requests.get(external_url, timeout=10)
+        response.raise_for_status()
+        markup = response.text
+        logger.debug(f"Fetched {len(markup)=} for entry.{pk=}...")
+        content = extract_content(markup)
+        logger.info(f"extracted content from {external_url=}, {markup[:100]=}")
+    except requests.RequestException as e:
+        logger.error(f"Error fetching content from {external_url=}: {e=}")
+        return
+
+    logger.info(f"dispatching summarize task for {pk=} with {content[:40]=}...")
+    summarize_content.apply_async(
+        (content, page.title, SUMMARIZATION_MODEL), link=save_page_summary_value.s(pk)
+    )
+
+
+@app.task
+def set_summary_for_video_page(pk: int):
+    logger.info("Summarization not implemented")
+
+
+@app.task
+def set_summary_for_poll_page(pk: int):
+    logger.info("Summarization not implemented")
+
+
+@app.task
+def set_thumbnail_for_video_page(pk: int):
+    from pages.models import PostPage
+
+    video = PostPage.objects.get(pk=pk)
+    set_video_thumbnail(video)

news/utils.py

@@ -12,16 +12,22 @@
 
 if typing.TYPE_CHECKING:
     from news.models import Video
+    from pages.models import PostPage
 
 
-def set_video_thumbnail(video: "Video"):
+def set_video_thumbnail(video: typing.Union["Video", "PostPage"]):
     """
     Given a video model, use oembed to fetch the thumbnail and save it to the model
     """
+    from news.models import Video
+    from pages.models import PostPage
+
     YOUTUBE_OEMBED_ENDPOINT = "https://www.youtube.com/oembed"
 
-    if not video.is_video:
+    if isinstance(video, Video) and not video.is_video:
         raise Exception(f"{video} is not a video, cannot set thumbnail.")
+    elif isinstance(video, PostPage) and not video.post_content_type == "Video":
+        raise Exception(f"{video}'s content is not a video, cannot set thumbnail.")
 
     url = YOUTUBE_OEMBED_ENDPOINT + f"?url={video.external_url}"
     response = requests.get(url)
@@ -40,7 +46,11 @@ def set_video_thumbnail(video: "Video"):
         width=json.get("thumbnail_width"),
         height=json.get("thumbnail_height"),
     )
-    video.thumbnail = image
+
+    if isinstance(video, Video):
+        video.thumbnail = image
+    elif isinstance(video, PostPage):
+        video.video_thumbnail = image
     video.save()