From fb67f913b58dd0c73ea5d4cf7e8c5defeecfe768 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 16 Dec 2025 10:33:42 -0600 Subject: [PATCH 1/7] feat: allow section-based Surrogate-Key to purge releases and other things without nukiung the whole cache --- pydotorg/middleware.py | 39 ++++++++++++++++++++++++++---- pydotorg/tests/test_middleware.py | 40 ++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/pydotorg/middleware.py b/pydotorg/middleware.py index 8f339647a..8dc33a8d3 100644 --- a/pydotorg/middleware.py +++ b/pydotorg/middleware.py @@ -19,17 +19,46 @@ def __call__(self, request): class GlobalSurrogateKey: - """Middleware to insert a Surrogate-Key for purging in Fastly or other caches.""" + """Middleware to insert a Surrogate-Key for purging in Fastly or other caches. + + Adds both a global key (for full site purges) and section-based keys + derived from the URL path (for targeted purges like /downloads/). + """ def __init__(self, get_response): """Store the get_response callable.""" self.get_response = get_response + def _get_section_key(self, path): + """Extract section surrogate key from URL path. + + Examples: + /downloads/ -> downloads + /downloads/release/python-3141/ -> downloads + /events/python-events/ -> events + / -> None + """ + parts = path.strip("/").split("/") + if parts and parts[0]: + return parts[0] + return None + def __call__(self, request): - """Append the global surrogate key to the response header.""" + """Append the global and section surrogate keys to the response header.""" response = self.get_response(request) + keys = [] if hasattr(settings, "GLOBAL_SURROGATE_KEY"): - response["Surrogate-Key"] = " ".join( - filter(None, [settings.GLOBAL_SURROGATE_KEY, response.get("Surrogate-Key")]) - ) + keys.append(settings.GLOBAL_SURROGATE_KEY) + + section_key = self._get_section_key(request.path) + if section_key: + keys.append(section_key) + + existing = response.get("Surrogate-Key") + if existing: + keys.append(existing) + + if keys: + response["Surrogate-Key"] = " ".join(keys) + return response diff --git a/pydotorg/tests/test_middleware.py b/pydotorg/tests/test_middleware.py index 17ff401c9..2ef81bca7 100644 --- a/pydotorg/tests/test_middleware.py +++ b/pydotorg/tests/test_middleware.py @@ -1,6 +1,8 @@ from django.contrib.redirects.models import Redirect from django.contrib.sites.models import Site -from django.test import TestCase +from django.test import TestCase, override_settings + +from pydotorg.middleware import GlobalSurrogateKey class MiddlewareTests(TestCase): @@ -21,3 +23,39 @@ def test_redirects(self): response = self.client.get(url) self.assertEqual(response.status_code, 301) self.assertEqual(response["Location"], redirect.new_path) + + +class GlobalSurrogateKeyTests(TestCase): + def test_get_section_key(self): + """Test section key extraction from URL paths.""" + middleware = GlobalSurrogateKey(lambda r: None) + + self.assertEqual(middleware._get_section_key("/downloads/"), "downloads") + self.assertEqual(middleware._get_section_key("/downloads/release/python-3141/"), "downloads") + self.assertEqual(middleware._get_section_key("/events/"), "events") + self.assertEqual(middleware._get_section_key("/events/python-events/123/"), "events") + self.assertEqual(middleware._get_section_key("/sponsors/"), "sponsors") + + # returns None + self.assertIsNone(middleware._get_section_key("/")) + + self.assertEqual(middleware._get_section_key("/downloads"), "downloads") + self.assertEqual(middleware._get_section_key("downloads/"), "downloads") + + @override_settings(GLOBAL_SURROGATE_KEY="pydotorg-app") + def test_surrogate_key_header_includes_section(self): + """Test that Surrogate-Key header includes both global and section keys.""" + response = self.client.get("/downloads/") + self.assertTrue(response.has_header("Surrogate-Key")) + surrogate_key = response["Surrogate-Key"] + + self.assertIn("pydotorg-app", surrogate_key) + self.assertIn("downloads", surrogate_key) + + @override_settings(GLOBAL_SURROGATE_KEY="pydotorg-app") + def test_surrogate_key_header_homepage(self): + """Test that homepage only has global surrogate key.""" + response = self.client.get("/") + self.assertTrue(response.has_header("Surrogate-Key")) + surrogate_key = response["Surrogate-Key"] + self.assertIn("pydotorg-app", surrogate_key) From c778fce2df22ffa852af6b41265c1a53e73d3197 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 16 Dec 2025 10:39:49 -0600 Subject: [PATCH 2/7] helper to purge key --- fastly/utils.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/fastly/utils.py b/fastly/utils.py index 6ff3217fa..e3c236418 100644 --- a/fastly/utils.py +++ b/fastly/utils.py @@ -19,3 +19,31 @@ def purge_url(path): ) return None + + +def purge_surrogate_key(key): + """ + Purge all Fastly cached content tagged with a surrogate key. + + Common keys (set by GlobalSurrogateKey middleware): + - 'pydotorg-app': Purges entire site + - 'downloads': Purges all /downloads/* pages + - 'events': Purges all /events/* pages + - 'sponsors': Purges all /sponsors/* pages + - etc. (first path segment becomes the surrogate key) + + Returns the response from Fastly API, or None if not configured. + """ + if settings.DEBUG: + return None + + api_key = getattr(settings, 'FASTLY_API_KEY', None) + service_id = getattr(settings, 'FASTLY_SERVICE_ID', None) + if not api_key or not service_id: + return None + + response = requests.post( + f'https://api.fastly.com/service/{service_id}/purge/{key}', + headers={'Fastly-Key': api_key}, + ) + return response From 71cc54f682fb37242ac500d38062f7a4985853f6 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 16 Dec 2025 10:39:56 -0600 Subject: [PATCH 3/7] need svc id --- pydotorg/settings/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pydotorg/settings/base.py b/pydotorg/settings/base.py index 344e8ae3d..81b3f4d3e 100644 --- a/pydotorg/settings/base.py +++ b/pydotorg/settings/base.py @@ -265,6 +265,7 @@ ### Fastly ### FASTLY_API_KEY = False # Set to Fastly API key in production to allow pages to # be purged on save +FASTLY_SERVICE_ID = config("FASTLY_SERVICE_ID", default=None) # Required for surrogate key purging # Jobs JOB_THRESHOLD_DAYS = 90 From 5cfe8757be73c89b5809b94c87322ff97a259ad1 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 16 Dec 2025 10:40:13 -0600 Subject: [PATCH 4/7] use our new fancy helper to clean up this signal --- apps/downloads/models.py | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/apps/downloads/models.py b/apps/downloads/models.py index c73a34f29..682a10caa 100644 --- a/apps/downloads/models.py +++ b/apps/downloads/models.py @@ -16,7 +16,7 @@ from apps.cms.models import ContentManageable, NameSlugModel from apps.downloads.managers import ReleaseManager from apps.pages.models import Page -from fastly.utils import purge_url +from fastly.utils import purge_surrogate_key, purge_url DEFAULT_MARKUP_TYPE = getattr(settings, "DEFAULT_MARKUP_TYPE", "markdown") @@ -288,38 +288,30 @@ def promote_latest_release(sender, instance, **kwargs): @receiver(post_save, sender=Release) def purge_fastly_download_pages(sender, instance, **kwargs): - """Purge Fastly caches so new Downloads show up more quickly.""" + """Purge Fastly caches so new Downloads show up more quickly. + + Uses surrogate key purging to clear ALL pages under /downloads/ in one request, + including dynamically added pages like /downloads/android/, /downloads/ios/, etc. + Falls back to individual URL purges if surrogate key purging is not configured. + """ # Don't purge on fixture loads if kwargs.get("raw", False): return # Only purge on published instances if instance.is_published: - # Purge our common pages - purge_url("/downloads/") - purge_url("/downloads/feed.rss") - purge_url("/downloads/latest/python2/") - purge_url("/downloads/latest/python3/") - # Purge minor version specific URLs (like /downloads/latest/python3.14/) - version = instance.get_version() - if instance.version == Release.PYTHON3 and version: - match = re.match(r"^3\.(\d+)", version) - if match: - purge_url(f"/downloads/latest/python3.{match.group(1)}/") - purge_url("/downloads/latest/prerelease/") - purge_url("/downloads/latest/pymanager/") - purge_url("/downloads/macos/") - purge_url("/downloads/source/") - purge_url("/downloads/windows/") + # Purge all /downloads/* pages via surrogate key (preferred method) + # This catches everything: /downloads/android/, /downloads/release/*, etc. + purge_surrogate_key("downloads") + + # Also purge related pages outside /downloads/ purge_url("/ftp/python/") if instance.get_version(): purge_url(f"/ftp/python/{instance.get_version()}/") - # See issue #584 for details + # See issue #584 for details - these are under /box/, not /downloads/ purge_url("/box/supernav-python-downloads/") purge_url("/box/homepage-downloads/") purge_url("/box/download-sources/") - # Purge the release page itself - purge_url(instance.get_absolute_url()) @receiver(post_save, sender=Release) From 7d5ba7bf5a34f8cdc0c79b57cb0a23c00698aa78 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 16 Dec 2025 11:06:29 -0600 Subject: [PATCH 5/7] (maybe) add purge action --- .github/workflows/purge-cache.yml | 28 ++++++++++++++++++++++++++++ apps/downloads/models.py | 7 ++++--- fastly/utils.py | 15 +++++++-------- pydotorg/middleware.py | 5 +++-- pydotorg/tests/test_middleware.py | 24 ++++++++++++------------ 5 files changed, 54 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/purge-cache.yml diff --git a/.github/workflows/purge-cache.yml b/.github/workflows/purge-cache.yml new file mode 100644 index 000000000..91f8b101e --- /dev/null +++ b/.github/workflows/purge-cache.yml @@ -0,0 +1,28 @@ +name: Purge Fastly Cache + +on: + push: + branches: [main] + paths: ['static/**', 'templates/**', '**/templates/**'] + workflow_dispatch: + inputs: + target: + description: 'Surrogate key to purge' + required: true + default: 'pydotorg-app' + type: choice + options: [pydotorg-app, downloads, events, sponsors, jobs] + +permissions: {} + +jobs: + purge: + runs-on: ubuntu-latest + env: + KEY: ${{ inputs.target || 'pydotorg-app' }} + steps: + - name: Purge ${{ env.KEY }} + run: | + curl -fsS -X POST \ + "https://api.fastly.com/service/${{ secrets.FASTLY_SERVICE_ID }}/purge/${{ env.KEY }}" \ + -H "Fastly-Key: ${{ secrets.FASTLY_API_KEY }}" diff --git a/apps/downloads/models.py b/apps/downloads/models.py index 682a10caa..b30b283e9 100644 --- a/apps/downloads/models.py +++ b/apps/downloads/models.py @@ -290,9 +290,10 @@ def promote_latest_release(sender, instance, **kwargs): def purge_fastly_download_pages(sender, instance, **kwargs): """Purge Fastly caches so new Downloads show up more quickly. - Uses surrogate key purging to clear ALL pages under /downloads/ in one request, - including dynamically added pages like /downloads/android/, /downloads/ios/, etc. - Falls back to individual URL purges if surrogate key purging is not configured. + Uses surrogate key purging to attempt to clear ALL pages under /downloads/ + in one request, including dynamically added pages like /downloads/android/, + /downloads/ios/, etc. Independently purges a set of specific non-/downloads/ + URLs via individual URL purges. """ # Don't purge on fixture loads if kwargs.get("raw", False): diff --git a/fastly/utils.py b/fastly/utils.py index e3c236418..c9e0b5125 100644 --- a/fastly/utils.py +++ b/fastly/utils.py @@ -22,8 +22,7 @@ def purge_url(path): def purge_surrogate_key(key): - """ - Purge all Fastly cached content tagged with a surrogate key. + """Purge all Fastly cached content tagged with a surrogate key. Common keys (set by GlobalSurrogateKey middleware): - 'pydotorg-app': Purges entire site @@ -37,13 +36,13 @@ def purge_surrogate_key(key): if settings.DEBUG: return None - api_key = getattr(settings, 'FASTLY_API_KEY', None) - service_id = getattr(settings, 'FASTLY_SERVICE_ID', None) + api_key = getattr(settings, "FASTLY_API_KEY", None) + service_id = getattr(settings, "FASTLY_SERVICE_ID", None) if not api_key or not service_id: return None - response = requests.post( - f'https://api.fastly.com/service/{service_id}/purge/{key}', - headers={'Fastly-Key': api_key}, + return requests.post( + f"https://api.fastly.com/service/{service_id}/purge/{key}", + headers={"Fastly-Key": api_key}, + timeout=30, ) - return response diff --git a/pydotorg/middleware.py b/pydotorg/middleware.py index 8dc33a8d3..b495f3aeb 100644 --- a/pydotorg/middleware.py +++ b/pydotorg/middleware.py @@ -29,7 +29,7 @@ def __init__(self, get_response): """Store the get_response callable.""" self.get_response = get_response - def _get_section_key(self, path): + def get_section_key(self, path): """Extract section surrogate key from URL path. Examples: @@ -37,6 +37,7 @@ def _get_section_key(self, path): /downloads/release/python-3141/ -> downloads /events/python-events/ -> events / -> None + """ parts = path.strip("/").split("/") if parts and parts[0]: @@ -50,7 +51,7 @@ def __call__(self, request): if hasattr(settings, "GLOBAL_SURROGATE_KEY"): keys.append(settings.GLOBAL_SURROGATE_KEY) - section_key = self._get_section_key(request.path) + section_key = self.get_section_key(request.path) if section_key: keys.append(section_key) diff --git a/pydotorg/tests/test_middleware.py b/pydotorg/tests/test_middleware.py index 2ef81bca7..f8c962316 100644 --- a/pydotorg/tests/test_middleware.py +++ b/pydotorg/tests/test_middleware.py @@ -26,21 +26,21 @@ def test_redirects(self): class GlobalSurrogateKeyTests(TestCase): - def test_get_section_key(self): + def testget_section_key(self): """Test section key extraction from URL paths.""" - middleware = GlobalSurrogateKey(lambda r: None) + middleware = GlobalSurrogateKey(lambda _: None) - self.assertEqual(middleware._get_section_key("/downloads/"), "downloads") - self.assertEqual(middleware._get_section_key("/downloads/release/python-3141/"), "downloads") - self.assertEqual(middleware._get_section_key("/events/"), "events") - self.assertEqual(middleware._get_section_key("/events/python-events/123/"), "events") - self.assertEqual(middleware._get_section_key("/sponsors/"), "sponsors") + self.assertEqual(middleware.get_section_key("/downloads/"), "downloads") + self.assertEqual(middleware.get_section_key("/downloads/release/python-3141/"), "downloads") + self.assertEqual(middleware.get_section_key("/events/"), "events") + self.assertEqual(middleware.get_section_key("/events/python-events/123/"), "events") + self.assertEqual(middleware.get_section_key("/sponsors/"), "sponsors") - # returns None - self.assertIsNone(middleware._get_section_key("/")) + # returns None + self.assertIsNone(middleware.get_section_key("/")) - self.assertEqual(middleware._get_section_key("/downloads"), "downloads") - self.assertEqual(middleware._get_section_key("downloads/"), "downloads") + self.assertEqual(middleware.get_section_key("/downloads"), "downloads") + self.assertEqual(middleware.get_section_key("downloads/"), "downloads") @override_settings(GLOBAL_SURROGATE_KEY="pydotorg-app") def test_surrogate_key_header_includes_section(self): @@ -58,4 +58,4 @@ def test_surrogate_key_header_homepage(self): response = self.client.get("/") self.assertTrue(response.has_header("Surrogate-Key")) surrogate_key = response["Surrogate-Key"] - self.assertIn("pydotorg-app", surrogate_key) + self.assertEqual(surrogate_key, "pydotorg-app") From 9b8dbcfb243c114057198d840bcc51b3254bf7ab Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Wed, 18 Feb 2026 20:37:47 -0600 Subject: [PATCH 6/7] fix: rename testget_section_key to test_get_section_key Co-Authored-By: Claude Opus 4.6 --- pydotorg/tests/test_middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydotorg/tests/test_middleware.py b/pydotorg/tests/test_middleware.py index f8c962316..b4de012e3 100644 --- a/pydotorg/tests/test_middleware.py +++ b/pydotorg/tests/test_middleware.py @@ -26,7 +26,7 @@ def test_redirects(self): class GlobalSurrogateKeyTests(TestCase): - def testget_section_key(self): + def test_get_section_key(self): """Test section key extraction from URL paths.""" middleware = GlobalSurrogateKey(lambda _: None) From b3770d3cbb14280666e1b9ea23a408ae31f7cfa9 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Wed, 18 Feb 2026 20:37:52 -0600 Subject: [PATCH 7/7] fix: add fallback to purge_url when FASTLY_SERVICE_ID is not configured Prevents silent no-op regression when surrogate key purging is not available. Co-Authored-By: Claude Opus 4.6 --- apps/downloads/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/downloads/models.py b/apps/downloads/models.py index b30b283e9..44b8c5f66 100644 --- a/apps/downloads/models.py +++ b/apps/downloads/models.py @@ -303,7 +303,11 @@ def purge_fastly_download_pages(sender, instance, **kwargs): if instance.is_published: # Purge all /downloads/* pages via surrogate key (preferred method) # This catches everything: /downloads/android/, /downloads/release/*, etc. - purge_surrogate_key("downloads") + # Falls back to purge_url if FASTLY_SERVICE_ID is not configured. + if getattr(settings, "FASTLY_SERVICE_ID", None): + purge_surrogate_key("downloads") + else: + purge_url("/downloads/") # Also purge related pages outside /downloads/ purge_url("/ftp/python/")