Fix URLs

Author: StephanAkkerman

src/tweet.py

@@ -54,6 +54,11 @@ class Tweet:
     hashtags: list[str]
     title: str
     media_types: list[str]
+    created_at: str = ""
+    likes: int = 0
+    retweets: int = 0
+    replies: int = 0
+    views: int = 0
 
     def to_dict(self) -> dict:
         """Serialize to a plain dict safe for JSON."""

src/xclient.py

@@ -139,6 +139,16 @@ def strip_trailing_tco(text: str) -> str:
     return _RE_TRAILING_TCO.sub("", text)
 
 
+def expand_tco_urls(text: str, url_entities: list[dict]) -> str:
+    """Replace t.co shortlinks with their expanded URLs using entities.urls."""
+    for entry in url_entities:
+        short = entry.get("url", "")
+        expanded = entry.get("expanded_url", "")
+        if short and expanded:
+            text = text.replace(short, expanded)
+    return text
+
+
 class XTimelineClient:
     """
     Minimal client for polling an X/Twitter timeline endpoint described by a cURL.
@@ -528,6 +538,8 @@ def _parse_single_tweet(
         # text & entities
         legacy = tw.get("legacy", {})
         text = legacy.get("full_text", "")
+        url_entities = legacy.get("entities", {}).get("urls", [])
+        text = expand_tco_urls(text, url_entities)
         text = unescape_entities(strip_trailing_tco(text))
 
         tickers = [t.upper() for t in self._entities(tw, "symbols") if t]
@@ -543,6 +555,19 @@ def _parse_single_tweet(
         user_img = self._user_field(tw, "profile_image_url_https")
         url = self._tweet_url(tid)
 
+        # engagement metrics
+        raw_created_at = legacy.get("created_at", "")
+        try:
+            created_at = dt.datetime.strptime(
+                raw_created_at, "%a %b %d %H:%M:%S +0000 %Y"
+            ).strftime("%Y-%m-%dT%H:%M:%SZ")
+        except (ValueError, TypeError):
+            created_at = raw_created_at
+        likes = int(legacy.get("favorite_count", 0) or 0)
+        retweets = int(legacy.get("retweet_count", 0) or 0)
+        replies = int(legacy.get("reply_count", 0) or 0)
+        views = int(tw.get("views", {}).get("count", 0) or 0)
+
         # media
         media_items, media_types = self._collect_media(tw)
 
@@ -615,6 +640,11 @@ def _parse_nested(n: dict) -> Tweet | None:
             hashtags=sorted(set(hashtags)),
             title=title,
             media_types=[m.type for m in uniq_media],
+            created_at=created_at,
+            likes=likes,
+            retweets=retweets,
+            replies=replies,
+            views=views,
         )
 
     # ---------- public APIs ----------

tests/test_xclient.py

@@ -9,7 +9,7 @@
 
 import pytest
 
-from xclient import XTimelineClient
+from xclient import XTimelineClient, expand_tco_urls
 
 # ---------------------------------------------------------------------------
 # Helpers
@@ -131,6 +131,22 @@ def _wrap_user(name: str, screen_name: str, image_url: str = "") -> dict:
     },
 }
 
+TWEET_WITH_METRICS: dict = {
+    "__typename": "Tweet",
+    "rest_id": "2039000000000000003",
+    "core": _wrap_user("Someone", "someone"),
+    "views": {"count": "24248", "state": "Enabled"},
+    "legacy": {
+        "id_str": "2039000000000000003",
+        "full_text": "Some tweet",
+        "created_at": "Wed Apr 01 19:15:49 +0000 2026",
+        "favorite_count": 120,
+        "retweet_count": 45,
+        "reply_count": 10,
+        "entities": {"hashtags": [], "symbols": []},
+    },
+}
+
 TWEET_WITH_TICKERS_AND_HASHTAGS: dict = {
     "__typename": "Tweet",
     "rest_id": "2039000000000000002",
@@ -297,6 +313,100 @@ def test_media_types_mirror_media(self, client):
 # Tickers and hashtags
 # ---------------------------------------------------------------------------
 
+class TestMetrics:
+    def test_created_at_parsed_to_iso(self, client):
+        t = _parse(client, TWEET_WITH_METRICS)
+        assert t.created_at == "2026-04-01T19:15:49Z"
+
+    def test_likes(self, client):
+        t = _parse(client, TWEET_WITH_METRICS)
+        assert t.likes == 120
+
+    def test_retweets(self, client):
+        t = _parse(client, TWEET_WITH_METRICS)
+        assert t.retweets == 45
+
+    def test_replies(self, client):
+        t = _parse(client, TWEET_WITH_METRICS)
+        assert t.replies == 10
+
+    def test_views(self, client):
+        t = _parse(client, TWEET_WITH_METRICS)
+        assert t.views == 24248
+
+    def test_missing_metrics_default_to_zero(self, client):
+        t = _parse(client, PLAIN_TWEET)
+        assert t.likes == 0
+        assert t.retweets == 0
+        assert t.views == 0
+
+    def test_missing_created_at_defaults_to_empty(self, client):
+        t = _parse(client, PLAIN_TWEET)
+        assert t.created_at == ""
+
+
+TWEET_WITH_URL: dict = {
+    "__typename": "Tweet",
+    "rest_id": "2039000000000000004",
+    "core": _wrap_user("FlappyBert", "flappybert"),
+    "legacy": {
+        "id_str": "2039000000000000004",
+        "full_text": "Play now 👉 https://t.co/tOjx6u4o0o",
+        "entities": {
+            "hashtags": [],
+            "symbols": [],
+            "urls": [
+                {
+                    "url": "https://t.co/tOjx6u4o0o",
+                    "expanded_url": "http://t.me/FlappyBertBot",
+                    "display_url": "t.me/FlappyBertBot",
+                }
+            ],
+        },
+    },
+}
+
+TWEET_WITH_MEDIA_URL: dict = {
+    "__typename": "Tweet",
+    "rest_id": "2039000000000000005",
+    "core": _wrap_user("Someone", "someone"),
+    "legacy": {
+        "id_str": "2039000000000000005",
+        "full_text": "Check this out https://t.co/medialink",
+        # media t.co links are NOT in entities.urls — only in entities.media
+        "entities": {"hashtags": [], "symbols": [], "urls": []},
+        "extended_entities": {
+            "media": [
+                {"media_url_https": "https://pbs.twimg.com/media/photo.jpg", "type": "photo"}
+            ]
+        },
+    },
+}
+
+
+class TestUrlExpansion:
+    def test_tco_replaced_with_expanded(self, client):
+        t = _parse(client, TWEET_WITH_URL)
+        assert "http://t.me/FlappyBertBot" in t.text
+        assert "t.co" not in t.text
+
+    def test_media_tco_still_stripped(self, client):
+        # media t.co has no entities.urls entry, so strip_trailing_tco removes it
+        t = _parse(client, TWEET_WITH_MEDIA_URL)
+        assert "t.co" not in t.text
+
+    def test_expand_tco_urls_replaces_all_occurrences(self):
+        entities = [{"url": "https://t.co/abc", "expanded_url": "https://example.com"}]
+        result = expand_tco_urls("see https://t.co/abc and https://t.co/abc", entities)
+        assert result == "see https://example.com and https://example.com"
+
+    def test_expand_tco_urls_ignores_missing_fields(self):
+        entities = [{"url": "", "expanded_url": "https://example.com"}, {"url": "https://t.co/abc"}]
+        # should not raise, text unchanged for invalid entries
+        result = expand_tco_urls("https://t.co/abc", entities)
+        assert result == "https://t.co/abc"
+
+
 class TestEntities:
     def test_ticker_uppercased(self, client):
         t = _parse(client, TWEET_WITH_TICKERS_AND_HASHTAGS)