fix: honor stored sourceUrl for mirror clone and permalinks (CM-1318)

Signed-off-by: Uroš Marolt <uros@marolt.me>

Author: themarolt

services/apps/mailing_list_integration/src/crowdmail/services/mirror/mirror_service.py

@@ -20,8 +20,6 @@
 from crowdmail.services.parse.noteren import get_email_from_git, git_run_command
 from crowdmail.settings import LORE_MIRROR_DIR
 
-LORE_BASE_URL = "https://lore.kernel.org/"
-
 RETRYABLE_MIRROR_ERRORS = (NetworkError, RateLimitError, RemoteServerError)
 
 retry_on_mirror_error = retry(
@@ -103,15 +101,14 @@ def list_mirror_dir(list_name: str, mirror_dir: str = LORE_MIRROR_DIR) -> str:
 
 
 @retry_on_mirror_error
-async def ensure_mirror(list_name: str, mirror_dir: str = LORE_MIRROR_DIR) -> str:
+async def ensure_mirror(list_name: str, source_url: str, mirror_dir: str = LORE_MIRROR_DIR) -> str:
     """Clone the list mirror if it's absent, otherwise fetch new commits. Returns the list dir."""
     list_dir = list_mirror_dir(list_name, mirror_dir)
     if not os.path.isdir(list_dir):
-        logger.info(f"Cloning lore list '{list_name}' into {list_dir}")
+        clone_url = source_url.rstrip("/") + "/"
+        logger.info(f"Cloning lore list '{list_name}' from {clone_url} into {list_dir}")
         os.makedirs(os.path.dirname(list_dir), exist_ok=True)
-        await _run_shell_command(
-            ["public-inbox-clone", "-q", f"{LORE_BASE_URL}{list_name}/", list_dir]
-        )
+        await _run_shell_command(["public-inbox-clone", "-q", clone_url, list_dir])
         # public-inbox-clone drops a manifest scoped to clone time; remove it
         # and re-fetch from inside list_dir so the manifest is regenerated
         # correctly there (needed to detect new epoch shards, e.g. git/1.git,

services/apps/mailing_list_integration/src/crowdmail/services/parse/noteren.py

@@ -355,7 +355,7 @@ def parse_email(
         "channel": channel,
         "title": subject,
         "body": json_body,
-        "url": "https://lore.kernel.org/r/" + msgid,
+        "url": source.rstrip("/") + "/r/" + msgid,
         "isContribution": True,
         "type": ActivityType.MESSAGE,
         "attributes": attributes,

services/apps/mailing_list_integration/src/crowdmail/worker/list_worker.py

@@ -81,7 +81,7 @@ async def _process_single_list(self, mailing_list: MailingList):
         state = ListState.FAILED
 
         try:
-            list_dir = await ensure_mirror(mailing_list.name)
+            list_dir = await ensure_mirror(mailing_list.name, mailing_list.source_url)
             heads = dict(mailing_list.last_processed_heads)
             activities_db = []
             activities_kafka = []

services/apps/mailing_list_integration/src/test/test_noteren.py

@@ -852,14 +852,14 @@ def test_parse_email_url_contains_message_id():
         b"body\n"
     )
     parsed = _parse_email(raw, "src", "chan", "c1", "b1")
-    assert parsed["activityData"]["url"] == "https://lore.kernel.org/r/unique-id@example.com"
+    assert parsed["activityData"]["url"] == "src/r/unique-id@example.com"
 
 
 def test_parse_email_url_with_no_message_id():
-    """When Message-ID is absent the url must end with a trailing slash and empty id."""
+    """When Message-ID is absent the url must end with a slash under the source's /r/ path."""
     raw = b"From: A <a@example.com>\nSubject: s\nDate: Mon, 1 Jan 2024 12:00:00 +0000\n\nbody\n"
     parsed = _parse_email(raw, "src", "chan", "c1", "b1")
-    assert parsed["activityData"]["url"] == "https://lore.kernel.org/r/"
+    assert parsed["activityData"]["url"] == "src/r/"
 
 
 def test_parse_email_result_structure():