Skip to content

Commit 488e407

Browse files
Benjamin-eecsjackwener
Benjamin-eecs
and
jackwener
authored
feat(twitter): add device-follow notification stream command
* feat(twitter): add device-follow command for /i/timeline notification stream (#1628) Closes #1628. Adds the twitter device-follow command, which reads the curated tweet list aggregated under a bell-icon "new posts from @usera and N others" notification. Direct GET /i/timeline redirects to /home, so the data is only reachable via the legacy v1.1 REST endpoint /i/api/2/notifications/device_follow.json , none of the existing twitter commands cover this stream: - twitter timeline home for-you / following feed (different endpoint) - twitter notifications the notification list itself, not aggregated tweets inside any one notification - twitter search search-based, can't reproduce the aggregation Endpoint discovery + field-mapping originally proposed by @traddo in #1628; this PR upstreams a clean implementation that: - Strategy.COOKIE + ct0 from CDP cookie jar + the public web bearer token from clis/twitter/utils.js (same auth path as twitter timeline) - Hits /i/api/2/notifications/device_follow.json directly via page.evaluate fetch on the x.com origin so SameSite=Lax cookies are preserved - Joins each entry.content.item.content.tweet.id to globalObjects.tweets[id] and resolves the author via globalObjects.users[tweet.user_id_str] - Returns the canonical twitter row columns (id, author, text, likes, retweets, replies, views, created_at, url), matching twitter timeline minus has_media / media_urls / card / quoted_tweet which the legacy v1.1 endpoint does not surface - Sets views: null rather than a 0 sentinel; the legacy endpoint does not return view counts even with include_ext_views=true, and the GraphQL TweetResultByRestId round-trip per tweet was judged too expensive for a list command (typed-errors §3: no scalar sentinels that lie about real engagement) - parseLimit enforces strict 1-200 integer validation with no silent clamping; the only baseline addition is the silent-sentinel on the "unknown" author fallback, which matches the exact precedent in twitter/timeline.js:76 that is already baselined Tests: 17 unit tests in device-follow.test.js cover parseLimit strict validation, URL parameter shape, entry/tweet join, user-resolution fallback, dedup via the seen set, empty-stream shape, the canonical column registration, AuthRequiredError on missing ct0, and CommandExecutionError on non-2xx fetch. Live verified the endpoint shape end-to-end against the logged-in session: HTTP 200 with the expected {globalObjects: {tweets, users}, timeline: {id: 'tweet_notifications', instructions: [{addEntries: {entries: []}}]}} envelope. The tester account has no bell-notification follows enabled, so entries is empty, but the shape and auth path are confirmed against the documented spec. * fix(twitter): harden device-follow typed boundaries * fix(twitter): fail fast on device-follow drift --------- Co-authored-by: jackwener <jakevingoo@gmail.com>
1 parent e682c1c commit 488e407

5 files changed

Lines changed: 525 additions & 4 deletions

File tree

‎cli-manifest.json‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23961,6 +23961,46 @@
2396123961
"sourceFile": "twitter/delete.js",
2396223962
"navigateBefore": true
2396323963
},
23964+
{
23965+
"site": "twitter",
23966+
"name": "device-follow",
23967+
"description": "Read the /i/timeline device-follow notification stream (tweets aggregated under a bell-icon \"new posts from @userA and N others\" notification)",
23968+
"access": "read",
23969+
"domain": "x.com",
23970+
"strategy": "cookie",
23971+
"browser": true,
23972+
"args": [
23973+
{
23974+
"name": "limit",
23975+
"type": "int",
23976+
"default": 20,
23977+
"required": false,
23978+
"help": "Maximum number of tweets to return (1-200, default 20)"
23979+
},
23980+
{
23981+
"name": "top-by-engagement",
23982+
"type": "int",
23983+
"default": 0,
23984+
"required": false,
23985+
"help": "When set to N>0, re-rank by weighted engagement and return the top N. Default 0 keeps upstream ordering."
23986+
}
23987+
],
23988+
"columns": [
23989+
"id",
23990+
"author",
23991+
"text",
23992+
"likes",
23993+
"retweets",
23994+
"replies",
23995+
"views",
23996+
"created_at",
23997+
"url"
23998+
],
23999+
"type": "js",
24000+
"modulePath": "twitter/device-follow.js",
24001+
"sourceFile": "twitter/device-follow.js",
24002+
"navigateBefore": "https://x.com"
24003+
},
2396424004
{
2396524005
"site": "twitter",
2396624006
"name": "download",

‎clis/twitter/device-follow.js‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
/**
2+
* Twitter `/i/timeline` device-follow notification stream, i.e. the
3+
* curated tweet list aggregated under a bell-icon "new posts from
4+
* @userA and N others" notification. Direct GET /i/timeline redirects
5+
* to /home; the data is only reachable via the legacy v1.1 REST
6+
* endpoint `/i/api/2/notifications/device_follow.json`.
7+
*
8+
* Endpoint discovery and field-mapping originally proposed by @traddo
9+
* in issue #1628.
10+
*/
11+
import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
12+
import { cli, Strategy } from '@jackwener/opencli/registry';
13+
import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js';
14+
15+
const DEVICE_FOLLOW_PATH = '/i/api/2/notifications/device_follow.json';
16+
const MAX_LIMIT = 200;
17+
18+
function parseLimit(value) {
19+
if (value === undefined || value === null || value === '') return 20;
20+
const limit = Number(value);
21+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) {
22+
throw new ArgumentError(`--limit must be an integer between 1 and ${MAX_LIMIT}`);
23+
}
24+
return limit;
25+
}
26+
27+
function buildDeviceFollowUrl(count) {
28+
const params = new URLSearchParams({
29+
include_profile_interstitial_type: '1',
30+
include_blocking: '1',
31+
include_blocked_by: '1',
32+
include_followed_by: '1',
33+
include_want_retweets: '1',
34+
include_mute_edge: '1',
35+
include_can_dm: '1',
36+
include_can_media_tag: '1',
37+
include_ext_has_nft_avatar: '1',
38+
include_ext_is_blue_verified: '1',
39+
include_ext_verified_type: '1',
40+
skip_status: '1',
41+
cards_platform: 'Web-12',
42+
include_cards: '1',
43+
include_ext_alt_text: 'true',
44+
include_quote_count: 'true',
45+
include_reply_count: '1',
46+
tweet_mode: 'extended',
47+
include_ext_views: 'true',
48+
count: String(count),
49+
});
50+
return `${DEVICE_FOLLOW_PATH}?${params.toString()}`;
51+
}
52+
53+
function extractEntries(timeline) {
54+
if (!timeline || !Array.isArray(timeline.instructions)) return null;
55+
const out = [];
56+
for (const inst of timeline.instructions) {
57+
const entries = inst?.addEntries?.entries;
58+
if (Array.isArray(entries)) out.push(...entries);
59+
}
60+
return out;
61+
}
62+
63+
function joinEntryToTweet(entry, tweets, users) {
64+
const tweetId = entry?.content?.item?.content?.tweet?.id;
65+
if (!tweetId) return null;
66+
const tw = tweets?.[tweetId];
67+
if (!tw) return null;
68+
const user = users?.[tw.user_id_str] || null;
69+
if (typeof user?.screen_name !== 'string' || !user.screen_name) return null;
70+
return { tweetId, tweet: tw, user };
71+
}
72+
73+
function shapeRow({ tweetId, tweet, user }) {
74+
const screenName = user.screen_name;
75+
return {
76+
id: tweetId,
77+
author: screenName,
78+
text: tweet?.full_text || tweet?.text || '',
79+
likes: tweet?.favorite_count ?? 0,
80+
retweets: tweet?.retweet_count ?? 0,
81+
replies: tweet?.reply_count ?? 0,
82+
// The legacy v1.1 endpoint does not return view counts even with
83+
// include_ext_views=true; surface null rather than a 0 sentinel
84+
// that would lie about real engagement (typed-errors §3).
85+
views: null,
86+
created_at: tweet?.created_at || '',
87+
url: `https://x.com/${screenName}/status/${tweetId}`,
88+
};
89+
}
90+
91+
function parseDeviceFollow(payload, seen) {
92+
if (!payload?.globalObjects || typeof payload.globalObjects !== 'object') return null;
93+
const tweets = payload?.globalObjects?.tweets || {};
94+
const users = payload?.globalObjects?.users || {};
95+
if (typeof tweets !== 'object' || typeof users !== 'object') return null;
96+
const entries = extractEntries(payload?.timeline);
97+
if (!entries) return null;
98+
const rows = [];
99+
let unmatchedTweetEntries = 0;
100+
let malformedEntries = 0;
101+
for (const entry of entries) {
102+
const hasTweetEntry = Boolean(entry?.content?.item?.content?.tweet?.id);
103+
if (!hasTweetEntry) {
104+
malformedEntries++;
105+
continue;
106+
}
107+
const joined = joinEntryToTweet(entry, tweets, users);
108+
if (!joined) {
109+
unmatchedTweetEntries++;
110+
continue;
111+
}
112+
if (seen.has(joined.tweetId)) continue;
113+
seen.add(joined.tweetId);
114+
rows.push(shapeRow(joined));
115+
}
116+
return { rows, entryCount: entries.length, unmatchedTweetEntries, malformedEntries };
117+
}
118+
119+
cli({
120+
site: 'twitter',
121+
name: 'device-follow',
122+
access: 'read',
123+
description: 'Read the /i/timeline device-follow notification stream (tweets aggregated under a bell-icon "new posts from @userA and N others" notification)',
124+
domain: 'x.com',
125+
strategy: Strategy.COOKIE,
126+
browser: true,
127+
args: [
128+
{ name: 'limit', type: 'int', default: 20, help: `Maximum number of tweets to return (1-${MAX_LIMIT}, default 20)` },
129+
{ name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank by weighted engagement and return the top N. Default 0 keeps upstream ordering.' },
130+
],
131+
columns: ['id', 'author', 'text', 'likes', 'retweets', 'replies', 'views', 'created_at', 'url'],
132+
func: async (page, kwargs) => {
133+
const limit = parseLimit(kwargs.limit);
134+
const cookies = await page.getCookies({ url: 'https://x.com' });
135+
const ct0 = cookies.find((c) => c.name === 'ct0')?.value || null;
136+
if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)');
137+
138+
const apiUrl = buildDeviceFollowUrl(limit);
139+
const headers = JSON.stringify({
140+
Authorization: `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`,
141+
'X-Csrf-Token': ct0,
142+
'X-Twitter-Auth-Type': 'OAuth2Session',
143+
'X-Twitter-Active-User': 'yes',
144+
});
145+
const data = await page.evaluate(`async () => {
146+
try {
147+
const r = await fetch("${apiUrl}", { method: "GET", headers: ${headers}, credentials: 'include' });
148+
if (!r.ok) return { error: r.status };
149+
try {
150+
return await r.json();
151+
} catch (e) {
152+
return { errorKind: 'non_json', detail: String(e && e.message || e) };
153+
}
154+
} catch (e) {
155+
return { errorKind: 'exception', detail: String(e && e.message || e) };
156+
}
157+
}`);
158+
if (data?.errorKind === 'non_json') {
159+
throw new CommandExecutionError(`Twitter device-follow returned non-JSON response: ${data.detail || 'unknown parse error'}`);
160+
}
161+
if (data?.errorKind === 'exception') {
162+
throw new CommandExecutionError(`Twitter device-follow fetch failed: ${data.detail || 'unknown error'}`);
163+
}
164+
if (data?.error) {
165+
if (data.error === 401 || data.error === 403) {
166+
throw new AuthRequiredError('x.com', `Twitter device-follow returned HTTP ${data.error}`);
167+
}
168+
throw new CommandExecutionError(`HTTP ${data.error}: Failed to fetch device-follow notification stream.`);
169+
}
170+
const parsed = parseDeviceFollow(data, new Set());
171+
if (!parsed) {
172+
throw new CommandExecutionError('Twitter device-follow response was missing the expected timeline/globalObjects shape.');
173+
}
174+
if (parsed.malformedEntries > 0 || parsed.unmatchedTweetEntries > 0) {
175+
throw new CommandExecutionError('Twitter device-follow entries could not be joined to tweet/user objects.');
176+
}
177+
if (parsed.rows.length === 0) {
178+
throw new EmptyResultError('twitter device-follow', 'No device-follow notification tweets found.');
179+
}
180+
const rows = parsed.rows;
181+
const trimmed = rows.slice(0, limit);
182+
return applyTopByEngagement(trimmed, kwargs['top-by-engagement']);
183+
},
184+
});
185+
186+
export const __test__ = {
187+
buildDeviceFollowUrl,
188+
extractEntries,
189+
joinEntryToTweet,
190+
shapeRow,
191+
parseDeviceFollow,
192+
parseLimit,
193+
};

0 commit comments

Comments
 (0)