Story #2435: Webpage Integration: Posts Ranking Algorithm (#2464)

Author: herzog0

ak/views.py

@@ -4,6 +4,7 @@
 from django.core.exceptions import PermissionDenied
 from django.http import Http404, HttpResponse
 from django.shortcuts import render
+from django.urls import reverse
 from django.views import View
 from django.views.generic import TemplateView
 
@@ -137,13 +138,33 @@ def get_v3_context_data(self, **kwargs):
             "primary_button_url": "www.example.com",
             "primary_button_label": "Explore Our History",
         }
+        tag_display = {"blogpost": "Blog"}
+        popular_entries = (
+            Entry.objects.ranked()
+            .filter(deleted_at__isnull=True, published=True)
+            .select_related("author")[:5]
+        )
         ctx["posts_from_the_boost_community"] = {
             "heading": "Posts from the Boost Community",
             "primary_cta_label": "View all posts",
-            "primary_cta_url": "#",
+            "primary_cta_url": reverse("news"),
             "variant": "card",
             "theme": "teal",
-            "items": SharedResources.demo_posts[:5],
+            "items": [
+                {
+                    "title": entry.title,
+                    "url": entry.get_absolute_url(),
+                    "date": entry.publish_at,
+                    "category": (
+                        tag_display.get(str(entry.tag).lower(), entry.tag.capitalize())
+                        if entry.tag
+                        else ""
+                    ),
+                    "tag": "",
+                    "author": entry.author.to_v3_profile_dict(),
+                }
+                for entry in popular_entries
+            ],
         }
         ctx["join_developers_building_the_future_of_cpp"] = {
             "items": SharedResources.demo_join_community_links[:5]

config/celery.py

@@ -118,3 +118,9 @@ def setup_periodic_tasks(sender, **kwargs):
         crontab(day_of_week="sun", hour=2, minute=0),
         app.signature("asciidoctor_sandbox.tasks.cleanup_old_sandbox_documents"),
     )
+
+    # Sync per-post page views from Plausible. Executes daily at 6:00 AM.
+    sender.add_periodic_task(
+        crontab(hour=6, minute=0),
+        app.signature("news.tasks.sync_post_views_from_plausible"),
+    )

config/settings.py

@@ -627,6 +627,7 @@
 BOOST_CALENDAR = "5rorfm42nvmpt77ac0vult9iig@group.calendar.google.com"
 CALENDAR_API_KEY = env("CALENDAR_API_KEY", default="changeme")
 PLAUSIBLE_STATS_KEY = env("PLAUSIBLE_STATS_KEY", default="changeme")
+POSTS_RANKING_GRAVITY = env.float("POSTS_RANKING_GRAVITY", default=2.0)
 EVENTS_CACHE_KEY = "homepage_events"
 EVENTS_CACHE_TIMEOUT = 300  # 5 min
 

docs/env_vars.md

@@ -94,3 +94,16 @@ This project uses environment variables to configure certain aspects of the appl
 - For **local development**, set this in your `.env` file. Note: docker compose only loads `env_file` at container creation, so after adding the variable run `docker compose up -d --force-recreate web celery-worker celery-beat` to pick it up.
 - In **deployed environments**, set as a kube secret in `kube/boost/values.yaml` (or the environment-specific yaml file).
 - Without this variable set, OpenRouter responds with `401 No cookie auth credentials found` and Celery retries the task up to 3 times before giving up.
+
+### `PLAUSIBLE_STATS_KEY`
+
+- API key for the Plausible Analytics API, used to sync per-post page view counts into `Entry.page_views`.
+- For **local development**, obtain a valid value from the Boost team.
+- In **deployed environments**, the valid value is set in the environment-specific secrets.
+- If not set (or set to `changeme`), the sync task is silently skipped.
+
+### `POSTS_RANKING_GRAVITY`
+
+- Controls the decay rate of the posts ranking algorithm (`score = views / (age_hours + 2) ^ gravity`).
+- Higher values cause recent posts to decay faster and drop in ranking sooner.
+- Defaults to `2.0` if not set.

env.template

@@ -51,6 +51,8 @@ CELERY_BROKER=redis://redis:6379/0
 CELERY_BACKEND=redis://redis:6379/0
 
 CALENDAR_API_KEY=changeme
+PLAUSIBLE_STATS_KEY=changeme
+POSTS_RANKING_GRAVITY=2.0
 
 # terraform vars for google cloud oauth credential creation for local dev use
 TF_VAR_google_cloud_email=

justfile

@@ -182,6 +182,9 @@ alias shell := console
 @manage +args:
     docker compose run --rm web python manage.py {{ args }}
 
+@sync_post_views *args:  ## sync per-post page view counts from Plausible (pass --dry-run to preview)
+    docker compose run --rm web python manage.py sync_post_views {{ args }}
+
 # Static File Management
 @down_sync_images:
     scripts/sync-large-static-images.sh --down-sync;

news/management/commands/sync_post_views.py

@@ -0,0 +1,36 @@
+import djclick as click
+import requests
+
+from news.plausible import fetch_post_views, update_page_views
+from news.models import Entry
+
+
+@click.command()
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    help="Show what would be updated without writing to the database",
+)
+def command(dry_run):
+    """Sync per-post page view counts from Plausible into Entry.page_views."""
+
+    try:
+        slug_views = fetch_post_views()
+    except (requests.HTTPError, ValueError) as e:
+        raise click.ClickException(str(e)) from e
+
+    if not slug_views:
+        click.echo("No matching post URLs returned by Plausible.")
+        return
+
+    entries = list(Entry.objects.filter(slug__in=slug_views.keys()))
+    matched = {e.slug: e for e in entries}
+
+    if dry_run:
+        click.echo(f"Would update {len(matched)} entries (dry run):")
+        for slug, entry in matched.items():
+            click.echo(f"  {slug}: {entry.page_views} -> {slug_views[slug]}")
+        return
+
+    updated = update_page_views(slug_views, entries=entries)
+    click.echo(f"Updated page_views for {updated} entries.")

news/migrations/0014_entry_page_views.py

@@ -0,0 +1,16 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("news", "0013_video_thumbnail"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="entry",
+            name="page_views",
+            field=models.PositiveIntegerField(default=0),
+        ),
+    ]

news/models.py

@@ -1,9 +1,11 @@
 from pathlib import Path
 
 from structlog import get_logger
+from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.db import models
-from django.db.models import Case, Value, When
+from django.db.models import Case, ExpressionWrapper, FloatField, F, Func, Value, When
+from django.db.models.functions import Greatest, Now, Power
 from django.urls import reverse
 from django.utils.functional import cached_property
 from django.utils.text import slugify
@@ -26,6 +28,12 @@
 logger = get_logger(__name__)
 
 
+class ExtractEpoch(Func):
+    function = "EXTRACT"
+    template = "%(function)s(EPOCH FROM %(expressions)s)"
+    output_field = FloatField()
+
+
 class EntryManager(models.Manager):
     def get_queryset(self):
         result = (
@@ -55,6 +63,20 @@ def get_queryset(self):
     def published(self):
         return self.get_queryset().filter(published=True)
 
+    def ranked(self):
+        gravity = float(getattr(settings, "POSTS_RANKING_GRAVITY", 2.0))
+        age_in_hours = ExpressionWrapper(
+            Greatest(ExtractEpoch(Now() - F("publish_at")), Value(0.0)) / Value(3600.0),
+            output_field=FloatField(),
+        )
+        score = ExpressionWrapper(
+            F("page_views") / Power(age_in_hours + Value(2.0), Value(gravity)),
+            output_field=FloatField(),
+        )
+        return (
+            self.get_queryset().annotate(ranking_score=score).order_by("-ranking_score")
+        )
+
 
 class Entry(models.Model):
     """A news entry.
@@ -102,6 +124,7 @@ class AlreadyApprovedError(Exception):
         blank=True,
         related_name="deleted_entries",
     )
+    page_views = models.PositiveIntegerField(default=0)
 
     objects = EntryManager()
 

news/plausible.py

@@ -0,0 +1,81 @@
+import requests
+import structlog
+from django.conf import settings
+
+from news.models import Entry
+from reports.constants import WEB_ANALYTICS_API_URL_V2, WEB_ANALYTICS_DOMAIN
+
+logger = structlog.get_logger(__name__)
+
+NEWS_ENTRY_PREFIX = "/news/entry/"
+
+
+def fetch_post_views() -> dict[str, int]:
+    """Return a slug-to-pageviews mapping fetched from Plausible API v2.
+
+    Returns an empty dict if the API key is not configured or the response
+    contains no matching URLs.
+    Raises requests.HTTPError on a non-2xx response.
+    """
+    if not settings.PLAUSIBLE_STATS_KEY or settings.PLAUSIBLE_STATS_KEY == "changeme":
+        logger.info(
+            "fetch_post_views.skipped", reason="PLAUSIBLE_STATS_KEY not configured"
+        )
+        return {}
+
+    payload = {
+        "site_id": WEB_ANALYTICS_DOMAIN,
+        "metrics": ["pageviews"],
+        "dimensions": ["event:page"],
+        "filters": [["contains", "event:page", [NEWS_ENTRY_PREFIX]]],
+        "date_range": "all",
+    }
+    headers = {
+        "Content-Type": "application/json; charset=utf-8",
+        "Authorization": f"Bearer {settings.PLAUSIBLE_STATS_KEY}",
+    }
+
+    response = requests.post(
+        url=WEB_ANALYTICS_API_URL_V2, json=payload, headers=headers
+    )
+    if not response.ok:
+        raise requests.HTTPError(
+            f"Plausible API error {response.status_code}: {response.text}",
+            response=response,
+        )
+
+    data = response.json()
+    if not data or "results" not in data:
+        raise ValueError(f"Unexpected Plausible API response: {data}")
+
+    slug_views: dict[str, int] = {}
+    for result in data["results"]:
+        path = result["dimensions"][0]
+        if not path.startswith(NEWS_ENTRY_PREFIX):
+            continue
+        slug = path[len(NEWS_ENTRY_PREFIX) :].rstrip("/")
+        if slug:
+            slug_views[slug] = int(result["metrics"][0])
+
+    return slug_views
+
+
+def update_page_views(slug_views: dict[str, int], entries: list | None = None) -> int:
+    """Bulk-update Entry.page_views from a slug-to-count mapping.
+
+    Returns the number of entries updated.
+    """
+    if not slug_views:
+        return 0
+
+    if entries is None:
+        entries = list(Entry.objects.filter(slug__in=slug_views.keys()))
+    unmatched = set(slug_views) - {e.slug for e in entries}
+    if unmatched:
+        logger.warning("update_page_views.unmatched_slugs", slugs=sorted(unmatched))
+
+    for entry in entries:
+        entry.page_views = slug_views[entry.slug]
+
+    Entry.objects.bulk_update(entries, ["page_views"])
+    return len(entries)