diff --git a/cli-manifest.json b/cli-manifest.json index c235344f7..2e4b709a0 100644 --- a/cli-manifest.json +++ b/cli-manifest.json @@ -22229,10 +22229,76 @@ "sourceFile": "twitter/bookmark.js", "navigateBefore": true }, + { + "site": "twitter", + "name": "bookmark-folder", + "description": "Read the tweets inside a single Twitter/X bookmark folder. Get the folder id from `opencli twitter bookmark-folders`.", + "access": "read", + "domain": "x.com", + "strategy": "cookie", + "browser": true, + "args": [ + { + "name": "folder-id", + "type": "string", + "required": true, + "positional": true, + "help": "Folder id from `opencli twitter bookmark-folders`." + }, + { + "name": "limit", + "type": "int", + "default": 20, + "required": false, + "help": "Maximum number of bookmarks to return (default 20)." + }, + { + "name": "top-by-engagement", + "type": "int", + "default": 0, + "required": false, + "help": "When set to N>0, re-rank the folder by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API's native (saved-time) ordering." + } + ], + "columns": [ + "id", + "author", + "text", + "likes", + "retweets", + "bookmarks", + "created_at", + "url" + ], + "type": "js", + "modulePath": "twitter/bookmark-folder.js", + "sourceFile": "twitter/bookmark-folder.js", + "navigateBefore": "https://x.com" + }, + { + "site": "twitter", + "name": "bookmark-folders", + "description": "List your Twitter/X bookmark folders (the user-created collections under Bookmarks). Returns folder id, name, item count, and created_at.", + "access": "read", + "domain": "x.com", + "strategy": "cookie", + "browser": true, + "args": [], + "columns": [ + "id", + "name", + "items", + "created_at" + ], + "type": "js", + "modulePath": "twitter/bookmark-folders.js", + "sourceFile": "twitter/bookmark-folders.js", + "navigateBefore": "https://x.com" + }, { "site": "twitter", "name": "bookmarks", - "description": "Fetch Twitter/X bookmarks", + "description": "Fetch your Twitter/X bookmarks (the logged-in user's saved tweets, newest first)", "access": "read", "domain": "x.com", "strategy": "cookie", @@ -22243,7 +22309,14 @@ "type": "int", "default": 20, "required": false, - "help": "" + "help": "Maximum number of bookmarks to return (default 20)." + }, + { + "name": "top-by-engagement", + "type": "int", + "default": 0, + "required": false, + "help": "When set to N>0, re-rank the bookmarks by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API's native (saved-time) ordering." } ], "columns": [ @@ -22290,7 +22363,7 @@ { "site": "twitter", "name": "download", - "description": "下载 Twitter/X 媒体(图片和视频)", + "description": "Download Twitter/X media (images and videos). Provide either to scan a profile's media tab, or --tweet-url to download a single tweet.", "access": "read", "domain": "x.com", "strategy": "cookie", @@ -22301,27 +22374,27 @@ "type": "str", "required": false, "positional": true, - "help": "Twitter username (downloads from media tab)" + "help": "Twitter username (with or without @) to scan their /media tab. Either or --tweet-url is required." }, { "name": "tweet-url", "type": "str", "required": false, - "help": "Single tweet URL to download" + "help": "Single tweet URL to download. Use this OR , not both required at once." }, { "name": "limit", "type": "int", "default": 10, "required": false, - "help": "Number of tweets to scan" + "help": "Maximum number of media items to download when scanning a profile (default 10). Ignored when --tweet-url is used." }, { "name": "output", "type": "str", "default": "./twitter-downloads", "required": false, - "help": "Output directory" + "help": "Output directory (default ./twitter-downloads). A per-source subdir is created inside." } ], "columns": [ @@ -22364,7 +22437,7 @@ { "site": "twitter", "name": "followers", - "description": "Get accounts following a Twitter/X user", + "description": "Get accounts following a Twitter/X user (defaults to the logged-in user when no user is given)", "access": "read", "domain": "x.com", "strategy": "ui", @@ -22382,7 +22455,7 @@ "type": "int", "default": 50, "required": false, - "help": "" + "help": "Maximum number of follower rows to return (default 50). Must be a positive integer." } ], "columns": [ @@ -22398,7 +22471,7 @@ { "site": "twitter", "name": "following", - "description": "Get accounts a Twitter/X user is following", + "description": "Get accounts a Twitter/X user is following (defaults to the logged-in user when no user is given)", "access": "read", "domain": "x.com", "strategy": "cookie", @@ -22416,7 +22489,7 @@ "type": "int", "default": 50, "required": false, - "help": "" + "help": "Maximum number of following rows to return (default 50). Must be a positive integer." } ], "columns": [ @@ -22485,7 +22558,7 @@ { "site": "twitter", "name": "likes", - "description": "Fetch liked tweets of a Twitter user", + "description": "Fetch liked tweets of a Twitter user (defaults to the logged-in user when no username is given)", "access": "read", "domain": "x.com", "strategy": "cookie", @@ -22496,14 +22569,21 @@ "type": "string", "required": false, "positional": true, - "help": "Twitter screen name (without @). Defaults to logged-in user." + "help": "Twitter screen name (with or without @). Defaults to the logged-in user when omitted." }, { "name": "limit", "type": "int", "default": 20, "required": false, - "help": "" + "help": "Maximum number of liked tweets to return (default 20)." + }, + { + "name": "top-by-engagement", + "type": "int", + "default": 0, + "required": false, + "help": "When set to N>0, re-rank the liked tweets by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API's native (recency) ordering." } ], "columns": [ @@ -22617,6 +22697,13 @@ "default": 50, "required": false, "help": "" + }, + { + "name": "top-by-engagement", + "type": "int", + "default": 0, + "required": false, + "help": "When set to N>0, re-rank the list timeline by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the list's native (recency) ordering." } ], "columns": [ @@ -22648,7 +22735,7 @@ "type": "int", "default": 50, "required": false, - "help": "" + "help": "Maximum number of lists to return (default 50)." } ], "columns": [ @@ -22666,7 +22753,7 @@ { "site": "twitter", "name": "notifications", - "description": "Get Twitter/X notifications", + "description": "Get your Twitter/X notifications (the logged-in user's likes/replies/follows feed, newest first)", "access": "read", "domain": "x.com", "strategy": "intercept", @@ -22677,7 +22764,7 @@ "type": "int", "default": 20, "required": false, - "help": "" + "help": "Maximum number of notifications to return (default 20)." } ], "columns": [ @@ -22728,7 +22815,7 @@ { "site": "twitter", "name": "profile", - "description": "Fetch a Twitter user profile (bio, stats, etc.)", + "description": "Fetch a Twitter user profile — bio, stats, etc. (defaults to the logged-in user when no username is given)", "access": "read", "domain": "x.com", "strategy": "cookie", @@ -22739,7 +22826,7 @@ "type": "string", "required": false, "positional": true, - "help": "Twitter screen name (without @). Defaults to logged-in user." + "help": "Twitter screen name (with or without @). Defaults to the logged-in user when omitted." } ], "columns": [ @@ -22763,7 +22850,7 @@ { "site": "twitter", "name": "quote", - "description": "Quote-tweet a specific tweet with your own text", + "description": "Quote-tweet a specific tweet with your own text, optionally with a local or remote image", "access": "write", "domain": "x.com", "strategy": "ui", @@ -22782,6 +22869,18 @@ "required": true, "positional": true, "help": "The text content of your quote" + }, + { + "name": "image", + "type": "str", + "required": false, + "help": "Optional local image path to attach to the quote tweet" + }, + { + "name": "image-url", + "type": "str", + "required": false, + "help": "Optional remote image URL to download and attach to the quote tweet" } ], "columns": [ @@ -22918,7 +23017,7 @@ { "site": "twitter", "name": "search", - "description": "Search Twitter/X for tweets", + "description": "Search Twitter/X for tweets, with optional --from / --has / --exclude / --product filters mapped to X's search operators", "access": "read", "domain": "x.com", "strategy": "intercept", @@ -22929,25 +23028,75 @@ "type": "string", "required": true, "positional": true, - "help": "Twitter/X search query (operators like `from:` and `since:` are supported)" + "help": "Search query. Raw X operators (e.g. \"exact phrase\", #tag, OR, lang:en, since:YYYY-MM-DD, from:, since:) are passed through unchanged." }, { "name": "filter", "type": "string", "default": "top", "required": false, - "help": "", + "help": "Legacy alias for --product. Kept for backwards compatibility; if --product is set it wins.", "choices": [ "top", "live" ] }, + { + "name": "product", + "type": "string", + "required": false, + "help": "Which X search tab to read: top (default), live (Latest), photos, videos. Maps to the f= URL param.", + "choices": [ + "top", + "live", + "photos", + "videos" + ] + }, + { + "name": "from", + "type": "string", + "required": false, + "help": "Restrict to tweets authored by . Leading @ is stripped. Equivalent to appending `from:` to the query." + }, + { + "name": "has", + "type": "string", + "required": false, + "help": "Restrict to tweets that have media|images|videos|links|replies. Maps to X's `filter:` operator.", + "choices": [ + "media", + "images", + "videos", + "links", + "replies" + ] + }, + { + "name": "exclude", + "type": "string", + "required": false, + "help": "Exclude tweets matching : replies|retweets|media|links. Maps to X's `-filter:` operator (retweets → -filter:nativeretweets).", + "choices": [ + "replies", + "retweets", + "media", + "links" + ] + }, { "name": "limit", "type": "int", "default": 15, "required": false, - "help": "" + "help": "Maximum number of tweets to return (default 15). Result count after server-side filtering." + }, + { + "name": "top-by-engagement", + "type": "int", + "default": 0, + "required": false, + "help": "When set to N>0, re-rank the results by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps X's native ordering." } ], "columns": [ @@ -22988,6 +23137,13 @@ "default": 50, "required": false, "help": "" + }, + { + "name": "top-by-engagement", + "type": "int", + "default": 0, + "required": false, + "help": "When set to N>0, re-rank the thread by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the conversation's structural ordering." } ], "columns": [ @@ -23008,7 +23164,7 @@ { "site": "twitter", "name": "timeline", - "description": "Fetch Twitter timeline (for-you or following)", + "description": "Fetch the logged-in user's home timeline (for-you algorithmic feed by default; pass --type following for the chronological feed of accounts you follow)", "access": "read", "domain": "x.com", "strategy": "cookie", @@ -23019,7 +23175,7 @@ "type": "str", "default": "for-you", "required": false, - "help": "Timeline type: for-you (algorithmic) or following (chronological)", + "help": "Which home-timeline feed to read. Default for-you (algorithmic). Use following for the chronological feed of accounts you follow.", "choices": [ "for-you", "following" @@ -23030,7 +23186,14 @@ "type": "int", "default": 20, "required": false, - "help": "" + "help": "Maximum number of tweets to return (default 20)." + }, + { + "name": "top-by-engagement", + "type": "int", + "default": 0, + "required": false, + "help": "When set to N>0, re-rank the timeline by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps X's native ordering." } ], "columns": [ @@ -23100,6 +23263,13 @@ "default": 20, "required": false, "help": "Max tweets to return" + }, + { + "name": "top-by-engagement", + "type": "int", + "default": 0, + "required": false, + "help": "When set to N>0, re-rank the tweets by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the chronological ordering." } ], "columns": [ diff --git a/clis/twitter/article.js b/clis/twitter/article.js index c6623942e..012107be5 100644 --- a/clis/twitter/article.js +++ b/clis/twitter/article.js @@ -1,6 +1,7 @@ import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; import { cli, Strategy } from '@jackwener/opencli/registry'; import { resolveTwitterQueryId } from './shared.js'; +import { TWITTER_BEARER_TOKEN } from './utils.js'; const TWEET_RESULT_BY_REST_ID_QUERY_ID = '7xflPyRiUxGVbJd4uWmbfg'; cli({ site: 'twitter', @@ -57,7 +58,7 @@ cli({ const ct0 = document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1]; if (!ct0) return {error: 'No ct0 cookie — not logged into x.com'}; - const bearer = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; + const bearer = ${JSON.stringify(TWITTER_BEARER_TOKEN)}; const headers = { 'Authorization': 'Bearer ' + decodeURIComponent(bearer), 'X-Csrf-Token': ct0, diff --git a/clis/twitter/bookmark-folder.js b/clis/twitter/bookmark-folder.js new file mode 100644 index 000000000..e792ad30e --- /dev/null +++ b/clis/twitter/bookmark-folder.js @@ -0,0 +1,189 @@ +import { cli, Strategy } from '@jackwener/opencli/registry'; +import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; +import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js'; +import { resolveTwitterQueryId } from './shared.js'; + +// Companion to bookmark-folders.js: reads tweets inside a single folder. +// X exposes folder contents through a separate timeline operation +// (BookmarkFolderTimeline). The shape is essentially the same as the +// global bookmarks timeline (bookmark_timeline_v2.timeline.instructions), +// just scoped to one folder via the bookmark_collection_id variable. +const OPERATION_NAME = 'BookmarkFolderTimeline'; +const FALLBACK_QUERY_ID = '13H7EUATwethsj_jZ6QQAQ'; +const FOLDER_ID_PATTERN = /^[A-Za-z0-9_-]+$/; + +const FEATURES = { + rweb_video_screen_enabled: false, + profile_label_improvements_pcf_label_in_post_enabled: true, + responsive_web_profile_redirect_enabled: false, + rweb_tipjar_consumption_enabled: false, + verified_phone_label_enabled: false, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + premium_content_api_read_enabled: false, + communities_web_enable_tweet_community_results_fetch: true, + c9s_tweet_anatomy_moderator_badge_enabled: true, + articles_preview_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: true, + tweet_awards_web_tipping_enabled: false, + content_disclosure_indicator_enabled: true, + content_disclosure_ai_generated_indicator_enabled: true, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: false, + responsive_web_enhance_cards_enabled: false, +}; + +function buildFolderTimelineUrl(queryId, folderId, count, cursor) { + const vars = { + bookmark_collection_id: String(folderId), + count, + includePromotedContent: false, + }; + if (cursor) vars.cursor = cursor; + return `/i/api/graphql/${queryId}/${OPERATION_NAME}` + + `?variables=${encodeURIComponent(JSON.stringify(vars))}` + + `&features=${encodeURIComponent(JSON.stringify(FEATURES))}`; +} + +function extractFolderTweet(result, seen) { + if (!result) return null; + const tw = result.tweet || result; + const legacy = tw.legacy || {}; + if (!tw.rest_id || seen.has(tw.rest_id)) return null; + seen.add(tw.rest_id); + const user = tw.core?.user_results?.result; + const screenName = user?.legacy?.screen_name || user?.core?.screen_name || ''; + const noteText = tw.note_tweet?.note_tweet_results?.result?.text; + return { + id: tw.rest_id, + author: screenName, + text: noteText || legacy.full_text || '', + likes: legacy.favorite_count || 0, + retweets: legacy.retweet_count || 0, + bookmarks: legacy.bookmark_count || 0, + created_at: legacy.created_at || '', + url: screenName ? `https://x.com/${screenName}/status/${tw.rest_id}` : `https://x.com/i/status/${tw.rest_id}`, + }; +} + +/** + * Parse the bookmark-folder timeline payload. Same instruction-walking + * pattern as the global bookmarks timeline; X re-uses the + * `bookmark_timeline_v2` envelope for folder-scoped queries. + * + * Exported via __test__. + */ +export function parseBookmarkFolderTimeline(data, seen) { + const tweets = []; + let nextCursor = null; + const instructions = data?.data?.bookmark_collection_timeline?.timeline?.instructions + || data?.data?.bookmark_timeline_v2?.timeline?.instructions + || data?.data?.bookmark_timeline?.timeline?.instructions + || []; + for (const inst of instructions) { + for (const entry of inst.entries || []) { + const content = entry.content; + if (content?.entryType === 'TimelineTimelineCursor' || content?.__typename === 'TimelineTimelineCursor') { + if (content.cursorType === 'Bottom' || content.cursorType === 'ShowMore') + nextCursor = content.value; + continue; + } + if (entry.entryId?.startsWith('cursor-bottom-') || entry.entryId?.startsWith('cursor-showMore-')) { + nextCursor = content?.value || content?.itemContent?.value || nextCursor; + continue; + } + const direct = extractFolderTweet(content?.itemContent?.tweet_results?.result, seen); + if (direct) { + tweets.push(direct); + continue; + } + for (const item of content?.items || []) { + const nested = extractFolderTweet(item.item?.itemContent?.tweet_results?.result, seen); + if (nested) tweets.push(nested); + } + } + } + return { tweets, nextCursor }; +} + +cli({ + site: 'twitter', + name: 'bookmark-folder', + access: 'read', + description: 'Read the tweets inside a single Twitter/X bookmark folder. Get the folder id from `opencli twitter bookmark-folders`.', + domain: 'x.com', + strategy: Strategy.COOKIE, + browser: true, + args: [ + { name: 'folder-id', positional: true, type: 'string', required: true, help: 'Folder id from `opencli twitter bookmark-folders`.' }, + { name: 'limit', type: 'int', default: 20, help: 'Maximum number of bookmarks to return (default 20).' }, + { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the folder by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API\'s native (saved-time) ordering.' }, + ], + columns: ['id', 'author', 'text', 'likes', 'retweets', 'bookmarks', 'created_at', 'url'], + func: async (page, kwargs) => { + const folderId = String(kwargs['folder-id'] || '').trim(); + if (!folderId || !FOLDER_ID_PATTERN.test(folderId)) { + throw new ArgumentError( + `Invalid folder-id: ${JSON.stringify(kwargs['folder-id'])}. Expected a safe folder ID from \`opencli twitter bookmark-folders\`.`, + ); + } + const limit = Number(kwargs.limit ?? 20); + if (!Number.isInteger(limit) || limit < 1) { + throw new ArgumentError(`Invalid --limit: ${JSON.stringify(kwargs.limit)}. Expected a positive integer.`); + } + + await page.goto('https://x.com'); + await page.wait(3); + const ct0 = await page.evaluate(`() => { + return document.cookie.split(';').map(c => c.trim()).find(c => c.startsWith('ct0='))?.split('=')[1] || null; + }`); + if (!ct0) + throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)'); + + const queryId = await resolveTwitterQueryId(page, OPERATION_NAME, FALLBACK_QUERY_ID); + + const headers = JSON.stringify({ + 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, + 'X-Csrf-Token': ct0, + 'X-Twitter-Auth-Type': 'OAuth2Session', + 'X-Twitter-Active-User': 'yes', + }); + + const allTweets = []; + const seen = new Set(); + let cursor = null; + for (let i = 0; i < 5 && allTweets.length < limit; i++) { + const fetchCount = Math.min(100, limit - allTweets.length + 10); + const apiUrl = buildFolderTimelineUrl(queryId, folderId, fetchCount, cursor); + const data = await page.evaluate(`async () => { + const r = await fetch(${JSON.stringify(apiUrl)}, { headers: ${headers}, credentials: 'include' }); + return r.ok ? await r.json() : { error: r.status }; + }`); + if (data?.error) { + if (allTweets.length === 0) + throw new CommandExecutionError(`HTTP ${data.error}: Failed to fetch folder ${folderId}. queryId may have expired, or the folder may not exist.`); + break; + } + const { tweets, nextCursor } = parseBookmarkFolderTimeline(data, seen); + allTweets.push(...tweets); + if (!nextCursor || nextCursor === cursor) break; + cursor = nextCursor; + } + const trimmed = allTweets.slice(0, limit); + return applyTopByEngagement(trimmed, kwargs['top-by-engagement']); + }, +}); + +export const __test__ = { + parseBookmarkFolderTimeline, + buildFolderTimelineUrl, + FOLDER_ID_PATTERN, +}; diff --git a/clis/twitter/bookmark-folder.test.js b/clis/twitter/bookmark-folder.test.js new file mode 100644 index 000000000..1a9f41c39 --- /dev/null +++ b/clis/twitter/bookmark-folder.test.js @@ -0,0 +1,334 @@ +import { describe, expect, it, vi } from 'vitest'; +import { getRegistry } from '@jackwener/opencli/registry'; +import { __test__ } from './bookmark-folder.js'; + +const { parseBookmarkFolderTimeline, buildFolderTimelineUrl, FOLDER_ID_PATTERN } = __test__; + +describe('twitter bookmark-folder URL builder', () => { + it('embeds the folder id and count in the variables payload', () => { + const url = buildFolderTimelineUrl('queryX', '12345', 50, null); + const m = url.match(/variables=([^&]+)/); + const vars = JSON.parse(decodeURIComponent(m[1])); + expect(vars.bookmark_collection_id).toBe('12345'); + expect(vars.count).toBe(50); + expect(vars.includePromotedContent).toBe(false); + expect(vars.cursor).toBeUndefined(); + }); + + it('appends the cursor when one is supplied', () => { + const url = buildFolderTimelineUrl('queryX', '12345', 50, 'CURSOR_VAL'); + const m = url.match(/variables=([^&]+)/); + const vars = JSON.parse(decodeURIComponent(m[1])); + expect(vars.cursor).toBe('CURSOR_VAL'); + }); + + it('coerces a numeric folder id to a string', () => { + const url = buildFolderTimelineUrl('queryX', 555, 10); + const m = url.match(/variables=([^&]+)/); + const vars = JSON.parse(decodeURIComponent(m[1])); + expect(vars.bookmark_collection_id).toBe('555'); + }); + + it('preserves opaque folder ids without truncating them', () => { + const url = buildFolderTimelineUrl('queryX', 'folder_AbC-123', 10); + const m = url.match(/variables=([^&]+)/); + const vars = JSON.parse(decodeURIComponent(m[1])); + expect(vars.bookmark_collection_id).toBe('folder_AbC-123'); + }); +}); + +describe('twitter bookmark-folder timeline parser', () => { + it('extracts tweets from bookmark_timeline_v2 envelope', () => { + const data = { + data: { + bookmark_timeline_v2: { + timeline: { + instructions: [ + { + type: 'TimelineAddEntries', + entries: [ + { + entryId: 'tweet-1', + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '1', + legacy: { + full_text: 'first folder tweet', + favorite_count: 9, + retweet_count: 2, + bookmark_count: 3, + created_at: 'Tue Mar 17 09:00:00 +0000 2026', + }, + core: { + user_results: { + result: { core: { screen_name: 'alice' } }, + }, + }, + }, + }, + }, + }, + }, + { + entryId: 'cursor-bottom-X', + content: { + __typename: 'TimelineTimelineCursor', + cursorType: 'Bottom', + value: 'NEXT_CURSOR', + }, + }, + ], + }, + ], + }, + }, + }, + }; + const { tweets, nextCursor } = parseBookmarkFolderTimeline(data, new Set()); + expect(tweets).toEqual([ + { + id: '1', + author: 'alice', + text: 'first folder tweet', + likes: 9, + retweets: 2, + bookmarks: 3, + created_at: 'Tue Mar 17 09:00:00 +0000 2026', + url: 'https://x.com/alice/status/1', + }, + ]); + expect(nextCursor).toBe('NEXT_CURSOR'); + }); + + it('falls back to bookmark_collection_timeline envelope', () => { + const data = { + data: { + bookmark_collection_timeline: { + timeline: { + instructions: [ + { + entries: [ + { + entryId: 'tweet-2', + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '2', + legacy: { full_text: 'collection envelope', favorite_count: 1, retweet_count: 0, bookmark_count: 0 }, + core: { user_results: { result: { legacy: { screen_name: 'bob' } } } }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }, + }; + const { tweets } = parseBookmarkFolderTimeline(data, new Set()); + expect(tweets).toHaveLength(1); + expect(tweets[0].id).toBe('2'); + expect(tweets[0].author).toBe('bob'); + }); + + it('uses note_tweet text when present (long-form tweets)', () => { + const data = { + data: { + bookmark_timeline_v2: { + timeline: { + instructions: [{ + entries: [{ + entryId: 'tweet-3', + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '3', + legacy: { full_text: 'short text', favorite_count: 0, retweet_count: 0, bookmark_count: 0 }, + note_tweet: { note_tweet_results: { result: { text: 'full long-form text' } } }, + core: { user_results: { result: { core: { screen_name: 'carol' } } } }, + }, + }, + }, + }, + }], + }], + }, + }, + }, + }; + const { tweets } = parseBookmarkFolderTimeline(data, new Set()); + expect(tweets[0].text).toBe('full long-form text'); + }); + + it('deduplicates tweets across the seen Set', () => { + const data = { + data: { + bookmark_timeline_v2: { + timeline: { + instructions: [{ + entries: [ + { + entryId: 'tweet-4', + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '4', + legacy: { full_text: 'first', favorite_count: 0, retweet_count: 0, bookmark_count: 0 }, + core: { user_results: { result: { core: { screen_name: 'dan' } } } }, + }, + }, + }, + }, + }, + { + entryId: 'tweet-4-dup', + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '4', + legacy: { full_text: 'duplicate' }, + core: { user_results: { result: { core: { screen_name: 'dan' } } } }, + }, + }, + }, + }, + }, + ], + }], + }, + }, + }, + }; + const seen = new Set(); + const { tweets } = parseBookmarkFolderTimeline(data, seen); + expect(tweets).toHaveLength(1); + expect(tweets[0].text).toBe('first'); + }); + + it('does not synthesize an unknown author sentinel when screen_name is missing', () => { + const data = { + data: { + bookmark_timeline_v2: { + timeline: { + instructions: [{ + entries: [{ + entryId: 'tweet-5', + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '5', + legacy: { full_text: 'missing author', favorite_count: 0, retweet_count: 0, bookmark_count: 0 }, + core: { user_results: { result: {} } }, + }, + }, + }, + }, + }], + }], + }, + }, + }, + }; + const { tweets } = parseBookmarkFolderTimeline(data, new Set()); + expect(tweets[0].author).toBe(''); + expect(tweets[0].url).toBe('https://x.com/i/status/5'); + }); + + it('returns empty array + null cursor for unknown envelope', () => { + expect(parseBookmarkFolderTimeline({}, new Set())).toEqual({ tweets: [], nextCursor: null }); + }); +}); + +describe('twitter bookmark-folder id validation', () => { + it('accepts numeric and opaque safe ids from bookmark-folders output', () => { + expect(FOLDER_ID_PATTERN.test('1234567890')).toBe(true); + expect(FOLDER_ID_PATTERN.test('folder_AbC-123')).toBe(true); + }); + + it('rejects ids that could pollute GraphQL variables or URL construction', () => { + for (const value of ['folder/123', 'folder?x=1', 'folder%2F123', 'folder.123', 'folder 123', '']) { + expect(FOLDER_ID_PATTERN.test(value)).toBe(false); + } + }); +}); + +describe('twitter bookmark-folder command (registry)', () => { + it('throws ArgumentError on unsafe folder-id before navigation', async () => { + const command = getRegistry().get('twitter/bookmark-folder'); + expect(command?.func).toBeTypeOf('function'); + const page = { + goto: vi.fn(), + wait: vi.fn(), + evaluate: vi.fn(), + }; + await expect(command.func(page, { 'folder-id': 'folder/123', limit: 5 })) + .rejects + .toThrow(/Invalid folder-id/); + expect(page.goto).not.toHaveBeenCalled(); + }); + + it('throws ArgumentError on empty folder-id', async () => { + const command = getRegistry().get('twitter/bookmark-folder'); + const page = { + goto: vi.fn(), + wait: vi.fn(), + evaluate: vi.fn(), + }; + await expect(command.func(page, { 'folder-id': ' ', limit: 5 })) + .rejects + .toThrow(/Invalid folder-id/); + }); + + it('throws ArgumentError on invalid limit before navigation', async () => { + const command = getRegistry().get('twitter/bookmark-folder'); + for (const limit of [0, -1, 1.5, Number.NaN]) { + const page = { + goto: vi.fn(), + wait: vi.fn(), + evaluate: vi.fn(), + }; + await expect(command.func(page, { 'folder-id': '12345', limit })) + .rejects + .toThrow(/Invalid --limit/); + expect(page.goto).not.toHaveBeenCalled(); + } + }); + + it('throws AuthRequiredError when ct0 cookie is missing', async () => { + const command = getRegistry().get('twitter/bookmark-folder'); + const page = { + goto: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + evaluate: vi.fn().mockResolvedValue(null), + }; + await expect(command.func(page, { 'folder-id': '12345', limit: 5 })) + .rejects + .toThrow(/Not logged into x.com/); + }); + + it('accepts an opaque safe folder-id and sends it in the GraphQL variables', async () => { + const command = getRegistry().get('twitter/bookmark-folder'); + const page = { + goto: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + evaluate: vi.fn() + .mockResolvedValueOnce('ct0-token') + .mockResolvedValueOnce('queryX') + .mockResolvedValueOnce({ data: { bookmark_timeline_v2: { timeline: { instructions: [] } } } }), + }; + const result = await command.func(page, { 'folder-id': 'folder_AbC-123', limit: 5 }); + expect(result).toEqual([]); + const fetchScript = page.evaluate.mock.calls[2][0]; + expect(decodeURIComponent(fetchScript)).toContain('"bookmark_collection_id":"folder_AbC-123"'); + }); +}); diff --git a/clis/twitter/bookmark-folders.js b/clis/twitter/bookmark-folders.js new file mode 100644 index 000000000..015bddf3b --- /dev/null +++ b/clis/twitter/bookmark-folders.js @@ -0,0 +1,117 @@ +import { cli, Strategy } from '@jackwener/opencli/registry'; +import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; +import { TWITTER_BEARER_TOKEN } from './utils.js'; +import { resolveTwitterQueryId } from './shared.js'; + +// X surfaces user-created bookmark folders through a GraphQL slice query. +// We mirror the patterns used in bookmarks.js / lists.js: a literal +// fallback queryId combined with a runtime lookup against the +// twitter-openapi placeholder.json so we keep working when X rotates IDs. +const OPERATION_NAME = 'bookmarkFoldersSlice'; +const FALLBACK_QUERY_ID = 'i78YDd0Tza-dWKw5H2Y7WA'; + +const FEATURES = { + rweb_tipjar_consumption_enabled: false, + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, +}; + +function buildUrl(queryId) { + const variables = JSON.stringify({}); + return `/i/api/graphql/${queryId}/${OPERATION_NAME}` + + `?variables=${encodeURIComponent(variables)}` + + `&features=${encodeURIComponent(JSON.stringify(FEATURES))}`; +} + +/** + * Walk the GraphQL response shape used by X's bookmark folders slice and + * project each folder onto our column row. + * + * X has shipped at least three different envelope shapes for this query + * across the last two years; the precedence order below preserves + * compatibility with older accounts whose Premium-eligibility flag is + * still on the legacy V2 envelope. + * + * Exported via __test__ so the parser is unit-testable without a browser. + */ +export function parseBookmarkFolders(data, seen) { + const folders = []; + const slice = data?.data?.viewer?.bookmark_collections_slice + || data?.data?.viewer_v2?.user_results?.result?.bookmark_collections_slice + || data?.data?.bookmark_collections_slice + || null; + const items = slice?.items || slice?.timeline?.timeline?.instructions?.flatMap?.(i => i.entries || []) || []; + for (const item of items) { + // Two known item shapes: direct {id, name, ...} (newer) or wrapped + // {content: {bookmarkCollectionResult: {...}}} (older / nested). + const folder + = item?.bookmarkCollection + || item?.content?.bookmarkCollection + || item?.content?.itemContent?.bookmark_collection + || item; + const id = folder?.id_str || folder?.id || folder?.rest_id || ''; + if (!id || seen.has(id)) continue; + seen.add(id); + const name = folder?.name || folder?.collection_name || ''; + // bookmarks_count is the X UI label; older envelopes used `count`. + const itemsCount = Number(folder?.bookmarks_count ?? folder?.items_count ?? folder?.count ?? 0) || 0; + const createdAt = folder?.created_at || folder?.timestamp_ms || ''; + folders.push({ + id: String(id), + name: String(name), + items: itemsCount, + created_at: String(createdAt), + }); + } + return folders; +} + +cli({ + site: 'twitter', + name: 'bookmark-folders', + access: 'read', + description: 'List your Twitter/X bookmark folders (the user-created collections under Bookmarks). Returns folder id, name, item count, and created_at.', + domain: 'x.com', + strategy: Strategy.COOKIE, + browser: true, + args: [], + columns: ['id', 'name', 'items', 'created_at'], + func: async (page) => { + await page.goto('https://x.com'); + await page.wait(3); + const ct0 = await page.evaluate(`() => { + return document.cookie.split(';').map(c => c.trim()).find(c => c.startsWith('ct0='))?.split('=')[1] || null; + }`); + if (!ct0) + throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)'); + + // Try the fa0311/twitter-openapi placeholder first; fall back to scraping + // client-web bundles for the queryId; final fallback is the pinned constant. + const queryId = await resolveTwitterQueryId(page, OPERATION_NAME, FALLBACK_QUERY_ID); + + const headers = JSON.stringify({ + 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, + 'X-Csrf-Token': ct0, + 'X-Twitter-Auth-Type': 'OAuth2Session', + 'X-Twitter-Active-User': 'yes', + }); + const apiUrl = buildUrl(queryId); + const data = await page.evaluate(`async () => { + const r = await fetch(${JSON.stringify(apiUrl)}, { headers: ${headers}, credentials: 'include' }); + return r.ok ? await r.json() : { error: r.status }; + }`); + if (data?.error) { + throw new CommandExecutionError(`HTTP ${data.error}: Failed to fetch bookmark folders. queryId may have expired, or your account may not have folder access.`); + } + const seen = new Set(); + return parseBookmarkFolders(data, seen); + }, +}); + +export const __test__ = { + parseBookmarkFolders, + buildUrl, +}; diff --git a/clis/twitter/bookmark-folders.test.js b/clis/twitter/bookmark-folders.test.js new file mode 100644 index 000000000..0b5afbbab --- /dev/null +++ b/clis/twitter/bookmark-folders.test.js @@ -0,0 +1,150 @@ +import { describe, expect, it, vi } from 'vitest'; +import { getRegistry } from '@jackwener/opencli/registry'; +import { __test__ } from './bookmark-folders.js'; + +const { parseBookmarkFolders, buildUrl } = __test__; + +describe('twitter bookmark-folders parser', () => { + it('returns [] for empty payload', () => { + expect(parseBookmarkFolders({}, new Set())).toEqual([]); + expect(parseBookmarkFolders({ data: {} }, new Set())).toEqual([]); + }); + + it('extracts folders from the modern viewer.bookmark_collections_slice envelope', () => { + const data = { + data: { + viewer: { + bookmark_collections_slice: { + items: [ + { + bookmarkCollection: { + id_str: '1234567890', + name: 'Reading list', + bookmarks_count: 42, + created_at: '2025-09-15T10:00:00.000Z', + }, + }, + { + bookmarkCollection: { + id_str: '9876543210', + name: 'Recipes', + bookmarks_count: 7, + created_at: '2026-01-03T03:14:00.000Z', + }, + }, + ], + }, + }, + }, + }; + expect(parseBookmarkFolders(data, new Set())).toEqual([ + { id: '1234567890', name: 'Reading list', items: 42, created_at: '2025-09-15T10:00:00.000Z' }, + { id: '9876543210', name: 'Recipes', items: 7, created_at: '2026-01-03T03:14:00.000Z' }, + ]); + }); + + it('falls back to legacy viewer_v2 envelope', () => { + const data = { + data: { + viewer_v2: { + user_results: { + result: { + bookmark_collections_slice: { + items: [{ bookmarkCollection: { id: 'abc', name: 'Old', count: 3 } }], + }, + }, + }, + }, + }, + }; + expect(parseBookmarkFolders(data, new Set())).toEqual([ + { id: 'abc', name: 'Old', items: 3, created_at: '' }, + ]); + }); + + it('falls back to flat bookmark_collections_slice envelope', () => { + const data = { + data: { + bookmark_collections_slice: { + items: [{ id_str: '5', name: 'Flat', bookmarks_count: 1, created_at: '2024-01-01' }], + }, + }, + }; + expect(parseBookmarkFolders(data, new Set())).toEqual([ + { id: '5', name: 'Flat', items: 1, created_at: '2024-01-01' }, + ]); + }); + + it('deduplicates folders by id across the seen Set', () => { + const data = { + data: { + viewer: { + bookmark_collections_slice: { + items: [ + { bookmarkCollection: { id_str: '1', name: 'A', bookmarks_count: 0 } }, + { bookmarkCollection: { id_str: '1', name: 'A again', bookmarks_count: 0 } }, + { bookmarkCollection: { id_str: '2', name: 'B', bookmarks_count: 0 } }, + ], + }, + }, + }, + }; + expect(parseBookmarkFolders(data, new Set())).toEqual([ + { id: '1', name: 'A', items: 0, created_at: '' }, + { id: '2', name: 'B', items: 0, created_at: '' }, + ]); + }); + + it('coerces missing items count to 0', () => { + const data = { + data: { + viewer: { + bookmark_collections_slice: { + items: [{ bookmarkCollection: { id: '1', name: 'No count' } }], + }, + }, + }, + }; + expect(parseBookmarkFolders(data, new Set())[0].items).toBe(0); + }); + + it('skips entries without an id', () => { + const data = { + data: { + viewer: { + bookmark_collections_slice: { + items: [ + { bookmarkCollection: { name: 'Anonymous' } }, + { bookmarkCollection: { id: '1', name: 'OK' } }, + ], + }, + }, + }, + }; + expect(parseBookmarkFolders(data, new Set())).toEqual([ + { id: '1', name: 'OK', items: 0, created_at: '' }, + ]); + }); +}); + +describe('twitter bookmark-folders URL builder', () => { + it('encodes the empty variables object and includes the queryId in the path', () => { + const url = buildUrl('queryid123'); + expect(url).toContain('/i/api/graphql/queryid123/bookmarkFoldersSlice'); + expect(url).toContain('variables=' + encodeURIComponent('{}')); + expect(url).toContain('features='); + }); +}); + +describe('twitter bookmark-folders command (registry)', () => { + it('throws AuthRequiredError when ct0 cookie is missing', async () => { + const command = getRegistry().get('twitter/bookmark-folders'); + expect(command?.func).toBeTypeOf('function'); + const page = { + goto: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + evaluate: vi.fn().mockResolvedValue(null), // null cookie → AuthRequired + }; + await expect(command.func(page, {})).rejects.toThrow(/Not logged into x.com/); + }); +}); diff --git a/clis/twitter/bookmark.js b/clis/twitter/bookmark.js index 3252081e2..c23f66776 100644 --- a/clis/twitter/bookmark.js +++ b/clis/twitter/bookmark.js @@ -1,5 +1,7 @@ import { CommandExecutionError } from '@jackwener/opencli/errors'; import { cli, Strategy } from '@jackwener/opencli/registry'; +import { parseTweetUrl, buildTwitterArticleScopeSource } from './shared.js'; + cli({ site: 'twitter', name: 'bookmark', @@ -15,22 +17,28 @@ cli({ func: async (page, kwargs) => { if (!page) throw new CommandExecutionError('Browser session required for twitter bookmark'); - await page.goto(kwargs.url); + const target = parseTweetUrl(kwargs.url); + await page.goto(target.url); await page.wait({ selector: '[data-testid="primaryColumn"]' }); const result = await page.evaluate(`(async () => { try { + ${buildTwitterArticleScopeSource(target.id)} + // Article-scoped: on conversation pages multiple bookmark/remove + // buttons render and a bare querySelector would silently bookmark + // a different tweet (e.g. the parent of the requested reply). let attempts = 0; let bookmarkBtn = null; let removeBtn = null; + let targetArticle = null; while (attempts < 20) { - // Check if already bookmarked - removeBtn = document.querySelector('[data-testid="removeBookmark"]'); + targetArticle = findTargetArticle(); + removeBtn = targetArticle?.querySelector('[data-testid="removeBookmark"]') || null; if (removeBtn) { return { ok: true, message: 'Tweet is already bookmarked.' }; } - bookmarkBtn = document.querySelector('[data-testid="bookmark"]'); + bookmarkBtn = targetArticle?.querySelector('[data-testid="bookmark"]') || null; if (bookmarkBtn) break; await new Promise(r => setTimeout(r, 500)); @@ -38,14 +46,15 @@ cli({ } if (!bookmarkBtn) { - return { ok: false, message: 'Could not find Bookmark button. Are you logged in?' }; + return { ok: false, message: 'Could not find Bookmark button on the requested tweet. Are you logged in?' }; } bookmarkBtn.click(); await new Promise(r => setTimeout(r, 1000)); // Verify - const verify = document.querySelector('[data-testid="removeBookmark"]'); + const verifyArticle = findTargetArticle() || targetArticle; + const verify = verifyArticle?.querySelector('[data-testid="removeBookmark"]'); if (verify) { return { ok: true, message: 'Tweet successfully bookmarked.' }; } else { diff --git a/clis/twitter/bookmark.test.js b/clis/twitter/bookmark.test.js new file mode 100644 index 000000000..c08852b40 --- /dev/null +++ b/clis/twitter/bookmark.test.js @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest'; +import { ArgumentError, CommandExecutionError } from '@jackwener/opencli/errors'; +import { getRegistry } from '@jackwener/opencli/registry'; +import './bookmark.js'; +import { createPageMock } from '../test-utils.js'; + +describe('twitter bookmark command', () => { + it('navigates to the tweet URL and reports success when the bookmark script confirms', async () => { + const cmd = getRegistry().get('twitter/bookmark'); + expect(cmd?.func).toBeTypeOf('function'); + const page = createPageMock([ + { ok: true, message: 'Tweet successfully bookmarked.' }, + ]); + const result = await cmd.func(page, { + url: 'https://x.com/alice/status/2040254679301718161', + }); + expect(page.goto).toHaveBeenCalledWith('https://x.com/alice/status/2040254679301718161'); + expect(page.wait).toHaveBeenNthCalledWith(1, { selector: '[data-testid="primaryColumn"]' }); + expect(page.wait).toHaveBeenNthCalledWith(2, 2); + const script = page.evaluate.mock.calls[0][0]; + // Idempotency probe: when already bookmarked ([data-testid="removeBookmark"] present), + // the script returns ok:true with an "already bookmarked" message. + expect(script).toContain("targetArticle?.querySelector('[data-testid=\"removeBookmark\"]')"); + expect(script).toContain("targetArticle?.querySelector('[data-testid=\"bookmark\"]')"); + expect(script).toContain('bookmarkBtn.click()'); + // Article scoping comes from the shared helper (buildTwitterArticleScopeSource): + // critical here because conversation pages render multiple + // bookmark/removeBookmark buttons and a bare querySelector would + // silently bookmark a different tweet. + expect(script).toContain('__twHasLinkToTarget'); + expect(script).toContain('__twGetStatusIdFromHref'); + expect(script).toContain("document.querySelectorAll('article')"); + expect(result).toEqual([ + { status: 'success', message: 'Tweet successfully bookmarked.' }, + ]); + }); + + it('returns a failed row without re-waiting when the bookmark script reports a UI mismatch', async () => { + const cmd = getRegistry().get('twitter/bookmark'); + const page = createPageMock([ + { + ok: false, + message: 'Could not find Bookmark button on the requested tweet. Are you logged in?', + }, + ]); + const result = await cmd.func(page, { + url: 'https://x.com/alice/status/2040254679301718161', + }); + expect(result).toEqual([ + { + status: 'failed', + message: 'Could not find Bookmark button on the requested tweet. Are you logged in?', + }, + ]); + expect(page.wait).toHaveBeenCalledTimes(1); + }); + + it('throws CommandExecutionError when no page is provided', async () => { + const cmd = getRegistry().get('twitter/bookmark'); + await expect(cmd.func(undefined, { + url: 'https://x.com/alice/status/2040254679301718161', + })).rejects.toThrow(CommandExecutionError); + }); + + it('rejects invalid tweet URLs before navigation', async () => { + const cmd = getRegistry().get('twitter/bookmark'); + const page = createPageMock([]); + await expect(cmd.func(page, { + url: 'https://evil.com/?next=https://x.com/alice/status/2040254679301718161', + })).rejects.toThrow(ArgumentError); + expect(page.goto).not.toHaveBeenCalled(); + expect(page.evaluate).not.toHaveBeenCalled(); + }); +}); diff --git a/clis/twitter/bookmarks.js b/clis/twitter/bookmarks.js index 5110e143b..3f4a7a609 100644 --- a/clis/twitter/bookmarks.js +++ b/clis/twitter/bookmarks.js @@ -1,6 +1,6 @@ import { cli, Strategy } from '@jackwener/opencli/registry'; import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; -const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; +import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js'; const BOOKMARKS_QUERY_ID = 'Fy0QMy4q_aZCpkO0PnyLYw'; const FEATURES = { rweb_video_screen_enabled: false, @@ -101,12 +101,13 @@ cli({ site: 'twitter', name: 'bookmarks', access: 'read', - description: 'Fetch Twitter/X bookmarks', + description: 'Fetch your Twitter/X bookmarks (the logged-in user\'s saved tweets, newest first)', domain: 'x.com', strategy: Strategy.COOKIE, browser: true, args: [ - { name: 'limit', type: 'int', default: 20 }, + { name: 'limit', type: 'int', default: 20, help: 'Maximum number of bookmarks to return (default 20).' }, + { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the bookmarks by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API\'s native (saved-time) ordering.' }, ], columns: ['id', 'author', 'text', 'likes', 'retweets', 'bookmarks', 'created_at', 'url'], func: async (page, kwargs) => { @@ -143,7 +144,7 @@ cli({ return null; }`) || BOOKMARKS_QUERY_ID; const headers = JSON.stringify({ - 'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`, + 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, 'X-Csrf-Token': ct0, 'X-Twitter-Auth-Type': 'OAuth2Session', 'X-Twitter-Active-User': 'yes', @@ -169,6 +170,7 @@ cli({ break; cursor = nextCursor; } - return allTweets.slice(0, limit); + const trimmed = allTweets.slice(0, limit); + return applyTopByEngagement(trimmed, kwargs['top-by-engagement']); }, }); diff --git a/clis/twitter/delete.js b/clis/twitter/delete.js index c6c798290..72e43caff 100644 --- a/clis/twitter/delete.js +++ b/clis/twitter/delete.js @@ -1,33 +1,13 @@ import { cli, Strategy } from '@jackwener/opencli/registry'; import { CommandExecutionError } from '@jackwener/opencli/errors'; -function extractTweetId(url) { - let pathname = ''; - try { - pathname = new URL(url).pathname; - } - catch { - throw new Error(`Invalid tweet URL: ${url}`); - } - const match = pathname.match(/\/status\/(\d+)/); - if (!match?.[1]) { - throw new Error(`Could not extract tweet ID from URL: ${url}`); - } - return match[1]; -} +import { parseTweetUrl, buildTwitterArticleScopeSource } from './shared.js'; + function buildDeleteScript(tweetId) { return `(async () => { try { const visible = (el) => !!el && (el.offsetParent !== null || el.getClientRects().length > 0); - const tweetId = ${JSON.stringify(tweetId)}; - const targetArticle = Array.from(document.querySelectorAll('article')).find((article) => - Array.from(article.querySelectorAll('a[href*="/status/"]')).some((link) => { - try { - return new URL(link.href, window.location.origin).pathname.includes('/status/' + tweetId); - } catch { - return false; - } - }) - ); + ${buildTwitterArticleScopeSource(tweetId)} + const targetArticle = findTargetArticle(); if (!targetArticle) { return { ok: false, message: 'Could not find the tweet card matching the requested URL.' }; @@ -82,17 +62,14 @@ cli({ func: async (page, kwargs) => { if (!page) throw new CommandExecutionError('Browser session required for twitter delete'); - let tweetId = ''; - try { - tweetId = extractTweetId(kwargs.url); - } - catch (error) { - const message = error instanceof Error ? error.message : String(error); - throw new CommandExecutionError(message); - } - await page.goto(kwargs.url); + // parseTweetUrl throws ArgumentError on malformed/off-domain inputs — + // this replaces the ad-hoc local extractTweetId which only checked + // the path shape and accepted any host (silent: would try to act on + // attacker-controlled redirect URLs). + const target = parseTweetUrl(kwargs.url); + await page.goto(target.url); await page.wait({ selector: '[data-testid="primaryColumn"]' }); // Wait for tweet to load completely - const result = await page.evaluate(buildDeleteScript(tweetId)); + const result = await page.evaluate(buildDeleteScript(target.id)); if (result.ok) { // Wait for the deletion request to be processed await page.wait(2); @@ -105,5 +82,4 @@ cli({ }); export const __test__ = { buildDeleteScript, - extractTweetId, }; diff --git a/clis/twitter/delete.test.js b/clis/twitter/delete.test.js index b71acbac2..c3854661b 100644 --- a/clis/twitter/delete.test.js +++ b/clis/twitter/delete.test.js @@ -1,13 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; -import { CommandExecutionError } from '@jackwener/opencli/errors'; +import { ArgumentError, CommandExecutionError } from '@jackwener/opencli/errors'; import { getRegistry } from '@jackwener/opencli/registry'; -import { __test__ } from './delete.js'; import './delete.js'; describe('twitter delete command', () => { - it('extracts tweet ids from both user and i/status URLs', () => { - expect(__test__.extractTweetId('https://x.com/alice/status/2040254679301718161?s=20')).toBe('2040254679301718161'); - expect(__test__.extractTweetId('https://x.com/i/status/2040318731105313143')).toBe('2040318731105313143'); - }); it('targets the matched tweet article instead of the first More button on the page', async () => { const cmd = getRegistry().get('twitter/delete'); expect(cmd?.func).toBeTypeOf('function'); @@ -23,9 +18,17 @@ describe('twitter delete command', () => { expect(page.wait).toHaveBeenNthCalledWith(1, { selector: '[data-testid="primaryColumn"]' }); expect(page.wait).toHaveBeenNthCalledWith(2, 2); const script = page.evaluate.mock.calls[0][0]; + // Article-scoping must come from the shared helper (not an inline + // `pathname.includes('/status/' + tweetId)` substring match — see + // codex-mini0 #1400 catch where `/status/123` would match + // `/status/1234567`). The helper emits `__twHasLinkToTarget` and + // `__twGetStatusIdFromHref` plus the canonical anchored regex. + expect(script).toContain('__twHasLinkToTarget'); + expect(script).toContain('__twGetStatusIdFromHref'); expect(script).toContain("document.querySelectorAll('article')"); - expect(script).toContain("'/status/' + tweetId"); expect(script).toContain("targetArticle.querySelectorAll('button,[role=\"button\"]')"); + // Substring match must NOT appear — exact-id match only. + expect(script).not.toContain("'/status/' + tweetId"); expect(result).toEqual([ { status: 'success', @@ -55,7 +58,7 @@ describe('twitter delete command', () => { ]); expect(page.wait).toHaveBeenCalledTimes(1); }); - it('normalizes invalid tweet URLs into CommandExecutionError', async () => { + it('rejects malformed or off-domain URLs with ArgumentError before navigation', async () => { const cmd = getRegistry().get('twitter/delete'); expect(cmd?.func).toBeTypeOf('function'); const page = { @@ -63,11 +66,20 @@ describe('twitter delete command', () => { wait: vi.fn(), evaluate: vi.fn(), }; + // parseTweetUrl bubbles ArgumentError directly (no CommandExecutionError + // wrapping); replaces the previous local extractTweetId path that hid + // typed-input failures behind a generic CliError. await expect(cmd.func(page, { url: 'https://x.com/alice/home', - })).rejects.toThrow(CommandExecutionError); + })).rejects.toThrow(ArgumentError); expect(page.goto).not.toHaveBeenCalled(); expect(page.wait).not.toHaveBeenCalled(); expect(page.evaluate).not.toHaveBeenCalled(); }); + it('throws CommandExecutionError when no page is provided', async () => { + const cmd = getRegistry().get('twitter/delete'); + await expect(cmd.func(undefined, { + url: 'https://x.com/alice/status/2040254679301718161', + })).rejects.toThrow(CommandExecutionError); + }); }); diff --git a/clis/twitter/download.js b/clis/twitter/download.js index 2194f4e1b..cc4e84982 100644 --- a/clis/twitter/download.js +++ b/clis/twitter/download.js @@ -12,14 +12,14 @@ cli({ site: 'twitter', name: 'download', access: 'read', - description: '下载 Twitter/X 媒体(图片和视频)', + description: 'Download Twitter/X media (images and videos). Provide either to scan a profile\'s media tab, or --tweet-url to download a single tweet.', domain: 'x.com', strategy: Strategy.COOKIE, args: [ - { name: 'username', positional: true, help: 'Twitter username (downloads from media tab)' }, - { name: 'tweet-url', help: 'Single tweet URL to download' }, - { name: 'limit', type: 'int', default: 10, help: 'Number of tweets to scan' }, - { name: 'output', default: './twitter-downloads', help: 'Output directory' }, + { name: 'username', positional: true, help: 'Twitter username (with or without @) to scan their /media tab. Either or --tweet-url is required.' }, + { name: 'tweet-url', help: 'Single tweet URL to download. Use this OR , not both required at once.' }, + { name: 'limit', type: 'int', default: 10, help: 'Maximum number of media items to download when scanning a profile (default 10). Ignored when --tweet-url is used.' }, + { name: 'output', default: './twitter-downloads', help: 'Output directory (default ./twitter-downloads). A per-source subdir is created inside.' }, ], columns: ['index', 'type', 'status', 'size'], func: async (page, kwargs) => { diff --git a/clis/twitter/followers.js b/clis/twitter/followers.js index 060361729..9b8629ef6 100644 --- a/clis/twitter/followers.js +++ b/clis/twitter/followers.js @@ -79,7 +79,7 @@ cli({ site: 'twitter', name: 'followers', access: 'read', - description: 'Get accounts following a Twitter/X user', + description: 'Get accounts following a Twitter/X user (defaults to the logged-in user when no user is given)', domain: 'x.com', strategy: Strategy.UI, browser: true, @@ -91,7 +91,7 @@ cli({ required: false, help: 'Twitter/X handle (with or without @). Omit to fetch followers of the currently logged-in account.', }, - { name: 'limit', type: 'int', default: 50 }, + { name: 'limit', type: 'int', default: 50, help: 'Maximum number of follower rows to return (default 50). Must be a positive integer.' }, ], // `followers` (count) is NOT exposed: the SPA followers-list view does not // render it. Use `twitter profile ` for per-user follower counts. diff --git a/clis/twitter/following.js b/clis/twitter/following.js index 5426f9b31..f56e75373 100644 --- a/clis/twitter/following.js +++ b/clis/twitter/following.js @@ -1,8 +1,8 @@ import { cli, Strategy } from '@jackwener/opencli/registry'; import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors'; import { resolveTwitterQueryId, sanitizeQueryId } from './shared.js'; +import { TWITTER_BEARER_TOKEN } from './utils.js'; -const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; const FOLLOWING_QUERY_ID = 'zx6e-TLzRkeDO_a7p4b3JQ'; // Following fallback const USER_BY_SCREEN_NAME_QUERY_ID = 'qRednkZG-rn1P6b48NINmQ'; @@ -135,7 +135,7 @@ cli({ site: 'twitter', name: 'following', access: 'read', - description: 'Get accounts a Twitter/X user is following', + description: 'Get accounts a Twitter/X user is following (defaults to the logged-in user when no user is given)', domain: 'x.com', strategy: Strategy.COOKIE, browser: true, @@ -147,7 +147,7 @@ cli({ required: false, help: 'Twitter/X handle (with or without @). Omit to fetch the accounts the currently logged-in user follows.', }, - { name: 'limit', type: 'int', default: 50 }, + { name: 'limit', type: 'int', default: 50, help: 'Maximum number of following rows to return (default 50). Must be a positive integer.' }, ], columns: ['screen_name', 'name', 'bio', 'followers'], func: async (page, kwargs) => { @@ -182,7 +182,7 @@ cli({ const followingQueryId = await resolveTwitterQueryId(page, 'Following', FOLLOWING_QUERY_ID); const userByScreenNameQueryId = await resolveTwitterQueryId(page, 'UserByScreenName', USER_BY_SCREEN_NAME_QUERY_ID); const headers = JSON.stringify({ - 'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`, + 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, 'X-Csrf-Token': ct0, 'X-Twitter-Auth-Type': 'OAuth2Session', 'X-Twitter-Active-User': 'yes', diff --git a/clis/twitter/hide-reply.js b/clis/twitter/hide-reply.js index 797faf4f7..d538413cf 100644 --- a/clis/twitter/hide-reply.js +++ b/clis/twitter/hide-reply.js @@ -1,5 +1,7 @@ import { CommandExecutionError } from '@jackwener/opencli/errors'; import { cli, Strategy } from '@jackwener/opencli/registry'; +import { parseTweetUrl, buildTwitterArticleScopeSource } from './shared.js'; + cli({ site: 'twitter', name: 'hide-reply', @@ -15,28 +17,45 @@ cli({ func: async (page, kwargs) => { if (!page) throw new CommandExecutionError('Browser session required for twitter hide-reply'); - await page.goto(kwargs.url); + const target = parseTweetUrl(kwargs.url); + await page.goto(target.url); await page.wait({ selector: '[data-testid="primaryColumn"]' }); const result = await page.evaluate(`(async () => { try { + ${buildTwitterArticleScopeSource(target.id)} + const visible = (el) => !!el && (el.offsetParent !== null || el.getClientRects().length > 0); + // Locate the article matching the requested status id, then find + // its More menu. Without article scoping we'd grab whatever the + // first "More" button on the page is — usually the parent tweet + // (silent: hide the wrong reply, or fail silently if the parent + // is not a reply you authored). let attempts = 0; + let targetArticle = null; let moreMenu = null; while (attempts < 20) { - moreMenu = document.querySelector('[aria-label="More"]'); - if (moreMenu) break; + targetArticle = findTargetArticle(); + if (targetArticle) { + const buttons = Array.from(targetArticle.querySelectorAll('button,[role="button"]')); + moreMenu = buttons.find((el) => visible(el) && (el.getAttribute('aria-label') || '').trim() === 'More'); + if (moreMenu) break; + } await new Promise(r => setTimeout(r, 500)); attempts++; } + if (!targetArticle) { + return { ok: false, message: 'Could not find the requested reply article on this page.' }; + } if (!moreMenu) { - return { ok: false, message: 'Could not find the "More" menu on this tweet. Are you logged in?' }; + return { ok: false, message: 'Could not find the "More" menu on the requested reply. Are you logged in?' }; } moreMenu.click(); await new Promise(r => setTimeout(r, 1000)); - // Look for the "Hide reply" menu item + // Look for the "Hide reply" menu item. Menu items render at the + // document root, not inside the article — scope is the open menu. const items = document.querySelectorAll('[role="menuitem"]'); let hideItem = null; for (const item of items) { diff --git a/clis/twitter/hide-reply.test.js b/clis/twitter/hide-reply.test.js new file mode 100644 index 000000000..6014c0bff --- /dev/null +++ b/clis/twitter/hide-reply.test.js @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest'; +import { ArgumentError, CommandExecutionError } from '@jackwener/opencli/errors'; +import { getRegistry } from '@jackwener/opencli/registry'; +import './hide-reply.js'; +import { createPageMock } from '../test-utils.js'; + +describe('twitter hide-reply command', () => { + it('navigates to the reply URL and reports success when the hide-reply script confirms', async () => { + const cmd = getRegistry().get('twitter/hide-reply'); + expect(cmd?.func).toBeTypeOf('function'); + const page = createPageMock([ + { ok: true, message: 'Reply successfully hidden.' }, + ]); + const result = await cmd.func(page, { + url: 'https://x.com/alice/status/2040254679301718161', + }); + expect(page.goto).toHaveBeenCalledWith('https://x.com/alice/status/2040254679301718161'); + expect(page.wait).toHaveBeenNthCalledWith(1, { selector: '[data-testid="primaryColumn"]' }); + expect(page.wait).toHaveBeenNthCalledWith(2, 2); + const script = page.evaluate.mock.calls[0][0]; + // Article-scoped More menu lookup — without scoping, the bare + // [aria-label="More"] selector grabs the parent tweet's More menu and + // silently hides the wrong reply (or fails because the parent is not a + // reply you authored). + expect(script).toContain('moreMenu.click()'); + expect(script).toContain('[role="menuitem"]'); + expect(script).toContain("'Hide reply'"); + expect(script).toContain('hideItem.click()'); + // Article scoping comes from the shared helper (buildTwitterArticleScopeSource): + // emits __twHasLinkToTarget + __twGetStatusIdFromHref + the anchored + // tweet-path regex. JSDOM-level coverage lives in shared.test.js. + expect(script).toContain('__twHasLinkToTarget'); + expect(script).toContain('__twGetStatusIdFromHref'); + expect(script).toContain("document.querySelectorAll('article')"); + expect(result).toEqual([ + { status: 'success', message: 'Reply successfully hidden.' }, + ]); + }); + + it('returns a failed row without re-waiting when the hide-reply script reports a UI mismatch', async () => { + const cmd = getRegistry().get('twitter/hide-reply'); + const page = createPageMock([ + { + ok: false, + message: 'Could not find "Hide reply" option. This may not be a reply on your tweet.', + }, + ]); + const result = await cmd.func(page, { + url: 'https://x.com/alice/status/2040254679301718161', + }); + expect(result).toEqual([ + { + status: 'failed', + message: 'Could not find "Hide reply" option. This may not be a reply on your tweet.', + }, + ]); + expect(page.wait).toHaveBeenCalledTimes(1); + }); + + it('throws CommandExecutionError when no page is provided', async () => { + const cmd = getRegistry().get('twitter/hide-reply'); + await expect(cmd.func(undefined, { + url: 'https://x.com/alice/status/2040254679301718161', + })).rejects.toThrow(CommandExecutionError); + }); + + it('rejects invalid tweet URLs before navigation', async () => { + const cmd = getRegistry().get('twitter/hide-reply'); + const page = createPageMock([]); + await expect(cmd.func(page, { + url: 'https://x.com.evil.com/alice/status/2040254679301718161', + })).rejects.toThrow(ArgumentError); + expect(page.goto).not.toHaveBeenCalled(); + expect(page.evaluate).not.toHaveBeenCalled(); + }); +}); diff --git a/clis/twitter/like.js b/clis/twitter/like.js index 59925af15..8f18cc84b 100644 --- a/clis/twitter/like.js +++ b/clis/twitter/like.js @@ -1,5 +1,7 @@ import { CommandExecutionError } from '@jackwener/opencli/errors'; import { cli, Strategy } from '@jackwener/opencli/registry'; +import { parseTweetUrl, buildTwitterArticleScopeSource } from './shared.js'; + cli({ site: 'twitter', name: 'like', @@ -15,21 +17,28 @@ cli({ func: async (page, kwargs) => { if (!page) throw new CommandExecutionError('Browser session required for twitter like'); - await page.goto(kwargs.url); + const target = parseTweetUrl(kwargs.url); + await page.goto(target.url); await page.wait({ selector: '[data-testid="primaryColumn"]' }); // Wait for tweet to load completely const result = await page.evaluate(`(async () => { try { - // Poll for the tweet to render + ${buildTwitterArticleScopeSource(target.id)} + // Poll for the tweet to render. We scope state probes to the + // article matching the requested status id — on conversation + // pages multiple articles render and a bare querySelector would + // grab the first one (silent: like the wrong tweet). let attempts = 0; let likeBtn = null; let unlikeBtn = null; - + let targetArticle = null; + while (attempts < 20) { - unlikeBtn = document.querySelector('[data-testid="unlike"]'); - likeBtn = document.querySelector('[data-testid="like"]'); - - if (unlikeBtn || likeBtn) break; - + targetArticle = findTargetArticle(); + likeBtn = targetArticle?.querySelector('[data-testid="like"]') || null; + unlikeBtn = targetArticle?.querySelector('[data-testid="unlike"]') || null; + + if (likeBtn || unlikeBtn) break; + await new Promise(r => setTimeout(r, 500)); attempts++; } @@ -46,9 +55,10 @@ cli({ // Click Like likeBtn.click(); await new Promise(r => setTimeout(r, 1000)); - - // Verify success by checking if the 'unlike' button appeared - const verifyBtn = document.querySelector('[data-testid="unlike"]'); + + // Verify success by checking if the 'unlike' button reappeared + const verifyArticle = findTargetArticle() || targetArticle; + const verifyBtn = verifyArticle?.querySelector('[data-testid="unlike"]'); if (verifyBtn) { return { ok: true, message: 'Tweet successfully liked.' }; } else { diff --git a/clis/twitter/like.test.js b/clis/twitter/like.test.js new file mode 100644 index 000000000..7a8692bef --- /dev/null +++ b/clis/twitter/like.test.js @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest'; +import { ArgumentError, CommandExecutionError } from '@jackwener/opencli/errors'; +import { getRegistry } from '@jackwener/opencli/registry'; +import './like.js'; +import { createPageMock } from '../test-utils.js'; + +describe('twitter like command', () => { + it('navigates to the tweet URL and reports success when the like script confirms', async () => { + const cmd = getRegistry().get('twitter/like'); + expect(cmd?.func).toBeTypeOf('function'); + const page = createPageMock([ + { ok: true, message: 'Tweet successfully liked.' }, + ]); + const result = await cmd.func(page, { + url: 'https://x.com/alice/status/2040254679301718161', + }); + expect(page.goto).toHaveBeenCalledWith('https://x.com/alice/status/2040254679301718161'); + expect(page.wait).toHaveBeenNthCalledWith(1, { selector: '[data-testid="primaryColumn"]' }); + expect(page.wait).toHaveBeenNthCalledWith(2, 2); + const script = page.evaluate.mock.calls[0][0]; + // Idempotency: looks for the unlike button (already-liked path) before clicking. + expect(script).toContain("targetArticle?.querySelector('[data-testid=\"like\"]')"); + expect(script).toContain("targetArticle?.querySelector('[data-testid=\"unlike\"]')"); + expect(script).toContain('likeBtn.click()'); + // Article scoping comes from the shared helper (buildTwitterArticleScopeSource): + // emits __twHasLinkToTarget + __twGetStatusIdFromHref + the anchored + // tweet-path regex. JSDOM-level coverage lives in shared.test.js. + expect(script).toContain('__twHasLinkToTarget'); + expect(script).toContain('__twGetStatusIdFromHref'); + expect(script).toContain("document.querySelectorAll('article')"); + expect(result).toEqual([ + { status: 'success', message: 'Tweet successfully liked.' }, + ]); + }); + + it('returns a failed row without re-waiting when the like script reports a UI mismatch', async () => { + const cmd = getRegistry().get('twitter/like'); + const page = createPageMock([ + { + ok: false, + message: 'Could not find the Like button on this tweet after waiting 10 seconds. Are you logged in?', + }, + ]); + const result = await cmd.func(page, { + url: 'https://x.com/alice/status/2040254679301718161', + }); + expect(result).toEqual([ + { + status: 'failed', + message: 'Could not find the Like button on this tweet after waiting 10 seconds. Are you logged in?', + }, + ]); + // Only the primaryColumn wait should run when ok is false. + expect(page.wait).toHaveBeenCalledTimes(1); + }); + + it('throws CommandExecutionError when no page is provided', async () => { + const cmd = getRegistry().get('twitter/like'); + await expect(cmd.func(undefined, { + url: 'https://x.com/alice/status/2040254679301718161', + })).rejects.toThrow(CommandExecutionError); + }); + + it('rejects invalid tweet URLs before navigation', async () => { + const cmd = getRegistry().get('twitter/like'); + const page = createPageMock([]); + await expect(cmd.func(page, { + url: 'https://x.com/alice/status/2040254679301718161/photo/1', + })).rejects.toThrow(ArgumentError); + expect(page.goto).not.toHaveBeenCalled(); + expect(page.evaluate).not.toHaveBeenCalled(); + }); +}); diff --git a/clis/twitter/likes.js b/clis/twitter/likes.js index 3e3ca9c1c..3761d1968 100644 --- a/clis/twitter/likes.js +++ b/clis/twitter/likes.js @@ -1,7 +1,7 @@ import { cli, Strategy } from '@jackwener/opencli/registry'; import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; import { resolveTwitterQueryId, sanitizeQueryId, extractMedia } from './shared.js'; -const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; +import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js'; const LIKES_QUERY_ID = 'RozQdCp4CilQzrcuU0NY5w'; const USER_BY_SCREEN_NAME_QUERY_ID = 'qRednkZG-rn1P6b48NINmQ'; const FEATURES = { @@ -138,13 +138,14 @@ cli({ site: 'twitter', name: 'likes', access: 'read', - description: 'Fetch liked tweets of a Twitter user', + description: 'Fetch liked tweets of a Twitter user (defaults to the logged-in user when no username is given)', domain: 'x.com', strategy: Strategy.COOKIE, browser: true, args: [ - { name: 'username', type: 'string', positional: true, help: 'Twitter screen name (without @). Defaults to logged-in user.' }, - { name: 'limit', type: 'int', default: 20 }, + { name: 'username', type: 'string', positional: true, help: 'Twitter screen name (with or without @). Defaults to the logged-in user when omitted.' }, + { name: 'limit', type: 'int', default: 20, help: 'Maximum number of liked tweets to return (default 20).' }, + { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the liked tweets by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API\'s native (recency) ordering.' }, ], columns: ['id', 'author', 'name', 'text', 'likes', 'retweets', 'created_at', 'url', 'has_media', 'media_urls'], func: async (page, kwargs) => { @@ -170,7 +171,7 @@ cli({ const likesQueryId = await resolveTwitterQueryId(page, 'Likes', LIKES_QUERY_ID); const userByScreenNameQueryId = await resolveTwitterQueryId(page, 'UserByScreenName', USER_BY_SCREEN_NAME_QUERY_ID); const headers = JSON.stringify({ - 'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`, + 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, 'X-Csrf-Token': ct0, 'X-Twitter-Auth-Type': 'OAuth2Session', 'X-Twitter-Active-User': 'yes', @@ -208,7 +209,8 @@ cli({ break; cursor = nextCursor; } - return allTweets.slice(0, limit); + const trimmed = allTweets.slice(0, limit); + return applyTopByEngagement(trimmed, kwargs['top-by-engagement']); }, }); export const __test__ = { diff --git a/clis/twitter/list-add.js b/clis/twitter/list-add.js index 8a53dd520..a2e45815c 100644 --- a/clis/twitter/list-add.js +++ b/clis/twitter/list-add.js @@ -2,8 +2,8 @@ import { cli, Strategy } from '@jackwener/opencli/registry'; import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; import { resolveTwitterQueryId } from './shared.js'; import { parseListsManagement } from './lists.js'; +import { TWITTER_BEARER_TOKEN } from './utils.js'; -const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; const USER_BY_SCREEN_NAME_QUERY_ID = 'qRednkZG-rn1P6b48NINmQ'; const LISTS_MANAGEMENT_QUERY_ID = '78UbkyXwXBD98IgUWXOy9g'; @@ -94,7 +94,7 @@ cli({ const userByScreenNameQueryId = await resolveTwitterQueryId(page, 'UserByScreenName', USER_BY_SCREEN_NAME_QUERY_ID); const headers = JSON.stringify({ - 'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`, + 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, 'X-Csrf-Token': ct0, 'X-Twitter-Auth-Type': 'OAuth2Session', 'X-Twitter-Active-User': 'yes', diff --git a/clis/twitter/list-remove.js b/clis/twitter/list-remove.js index cc3030912..36936c579 100644 --- a/clis/twitter/list-remove.js +++ b/clis/twitter/list-remove.js @@ -2,8 +2,8 @@ import { cli, Strategy } from '@jackwener/opencli/registry'; import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; import { resolveTwitterQueryId } from './shared.js'; import { parseListsManagement } from './lists.js'; +import { TWITTER_BEARER_TOKEN } from './utils.js'; -const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; const USER_BY_SCREEN_NAME_QUERY_ID = 'qRednkZG-rn1P6b48NINmQ'; const LISTS_MANAGEMENT_QUERY_ID = '78UbkyXwXBD98IgUWXOy9g'; @@ -101,7 +101,7 @@ cli({ const userByScreenNameQueryId = await resolveTwitterQueryId(page, 'UserByScreenName', USER_BY_SCREEN_NAME_QUERY_ID); const headers = JSON.stringify({ - 'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`, + 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, 'X-Csrf-Token': ct0, 'X-Twitter-Auth-Type': 'OAuth2Session', 'X-Twitter-Active-User': 'yes', diff --git a/clis/twitter/list-tweets.js b/clis/twitter/list-tweets.js index 2cc63bda6..20b8f35f3 100644 --- a/clis/twitter/list-tweets.js +++ b/clis/twitter/list-tweets.js @@ -1,7 +1,7 @@ import { cli, Strategy } from '@jackwener/opencli/registry'; import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; +import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js'; -const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; const LIST_TWEETS_QUERY_ID = 'RlZzktZY_9wJynoepm8ZsA'; const OPERATION_NAME = 'ListLatestTweetsTimeline'; @@ -115,6 +115,7 @@ cli({ args: [ { name: 'listId', positional: true, type: 'string', required: true, help: 'Numeric ID of a Twitter/X list (e.g. from `opencli twitter lists`)' }, { name: 'limit', type: 'int', default: 50 }, + { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the list timeline by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the list\'s native (recency) ordering.' }, ], columns: ['id', 'author', 'text', 'likes', 'retweets', 'replies', 'created_at', 'url'], func: async (page, kwargs) => { @@ -155,7 +156,7 @@ cli({ return null; }`) || LIST_TWEETS_QUERY_ID; const headers = JSON.stringify({ - 'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`, + 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, 'X-Csrf-Token': ct0, 'X-Twitter-Auth-Type': 'OAuth2Session', 'X-Twitter-Active-User': 'yes', @@ -181,6 +182,7 @@ cli({ break; cursor = nextCursor; } - return allTweets.slice(0, limit); + const trimmed = allTweets.slice(0, limit); + return applyTopByEngagement(trimmed, kwargs['top-by-engagement']); }, }); diff --git a/clis/twitter/lists.js b/clis/twitter/lists.js index d63fcbd18..61311dac5 100644 --- a/clis/twitter/lists.js +++ b/clis/twitter/lists.js @@ -1,7 +1,7 @@ import { cli, Strategy } from '@jackwener/opencli/registry'; import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; +import { TWITTER_BEARER_TOKEN } from './utils.js'; -const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; const LISTS_QUERY_ID = '78UbkyXwXBD98IgUWXOy9g'; const OPERATION_NAME = 'ListsManagementPageTimeline'; @@ -93,7 +93,7 @@ export const command = cli({ strategy: Strategy.COOKIE, browser: true, args: [ - { name: 'limit', type: 'int', default: 50 }, + { name: 'limit', type: 'int', default: 50, help: 'Maximum number of lists to return (default 50).' }, ], columns: ['id', 'name', 'members', 'followers', 'mode'], func: async (page, kwargs) => { @@ -130,7 +130,7 @@ export const command = cli({ return null; }`) || LISTS_QUERY_ID; const headers = JSON.stringify({ - 'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`, + 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, 'X-Csrf-Token': ct0, 'X-Twitter-Auth-Type': 'OAuth2Session', 'X-Twitter-Active-User': 'yes', diff --git a/clis/twitter/notifications.js b/clis/twitter/notifications.js index 8f862b973..e1bbd09b7 100644 --- a/clis/twitter/notifications.js +++ b/clis/twitter/notifications.js @@ -4,12 +4,12 @@ cli({ site: 'twitter', name: 'notifications', access: 'read', - description: 'Get Twitter/X notifications', + description: 'Get your Twitter/X notifications (the logged-in user\'s likes/replies/follows feed, newest first)', domain: 'x.com', strategy: Strategy.INTERCEPT, browser: true, args: [ - { name: 'limit', type: 'int', default: 20 }, + { name: 'limit', type: 'int', default: 20, help: 'Maximum number of notifications to return (default 20).' }, ], columns: ['id', 'action', 'author', 'text', 'url'], func: async (page, kwargs) => { diff --git a/clis/twitter/profile.js b/clis/twitter/profile.js index 893eddf9f..f6e47c9c7 100644 --- a/clis/twitter/profile.js +++ b/clis/twitter/profile.js @@ -1,17 +1,18 @@ import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; import { cli, Strategy } from '@jackwener/opencli/registry'; import { resolveTwitterQueryId } from './shared.js'; +import { TWITTER_BEARER_TOKEN } from './utils.js'; const USER_BY_SCREEN_NAME_QUERY_ID = 'qRednkZG-rn1P6b48NINmQ'; cli({ site: 'twitter', name: 'profile', access: 'read', - description: 'Fetch a Twitter user profile (bio, stats, etc.)', + description: 'Fetch a Twitter user profile — bio, stats, etc. (defaults to the logged-in user when no username is given)', domain: 'x.com', strategy: Strategy.COOKIE, browser: true, args: [ - { name: 'username', type: 'string', positional: true, help: 'Twitter screen name (without @). Defaults to logged-in user.' }, + { name: 'username', type: 'string', positional: true, help: 'Twitter screen name (with or without @). Defaults to the logged-in user when omitted.' }, ], columns: ['screen_name', 'name', 'bio', 'location', 'url', 'followers', 'following', 'tweets', 'likes', 'verified', 'created_at'], func: async (page, kwargs) => { @@ -38,7 +39,7 @@ cli({ const ct0 = document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1]; if (!ct0) return {error: 'No ct0 cookie — not logged into x.com'}; - const bearer = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; + const bearer = ${JSON.stringify(TWITTER_BEARER_TOKEN)}; const headers = { 'Authorization': 'Bearer ' + decodeURIComponent(bearer), 'X-Csrf-Token': ct0, diff --git a/clis/twitter/quote.js b/clis/twitter/quote.js index 28f927cfd..7c8a37ccf 100644 --- a/clis/twitter/quote.js +++ b/clis/twitter/quote.js @@ -1,10 +1,13 @@ +import * as fs from 'node:fs'; import { CommandExecutionError } from '@jackwener/opencli/errors'; import { cli, Strategy } from '@jackwener/opencli/registry'; -import { parseTweetUrl } from './shared.js'; - -function extractTweetId(url) { - return parseTweetUrl(url).id; -} +import { parseTweetUrl, buildTwitterArticleScopeSource } from './shared.js'; +import { + COMPOSER_FILE_INPUT_SELECTOR, + attachComposerImage, + downloadRemoteImage, + resolveImagePath, +} from './utils.js'; function buildQuoteComposerUrl(url) { // Twitter/X quote-tweet compose URL: the `url` param attaches the source @@ -17,15 +20,8 @@ function buildQuoteComposerUrl(url) { async function submitQuote(page, text, tweetId) { return page.evaluate(`(async () => { try { + ${buildTwitterArticleScopeSource(tweetId)} const visible = (el) => !!el && (el.offsetParent !== null || el.getClientRects().length > 0); - const getStatusId = (href) => { - try { - const match = new URL(href, window.location.origin).pathname.match(/^\\/(?:[^/]+|i)\\/status\\/(\\d+)\\/?$/); - return match?.[1] || null; - } catch { - return null; - } - }; const boxes = Array.from(document.querySelectorAll('[data-testid="tweetTextarea_0"]')); const box = boxes.find(visible) || boxes[0]; if (!box) { @@ -34,7 +30,6 @@ async function submitQuote(page, text, tweetId) { box.focus(); const textToInsert = ${JSON.stringify(text)}; - const tweetId = ${JSON.stringify(tweetId)}; // execCommand('insertText') is more reliable with Twitter's Draft.js editor. if (!document.execCommand('insertText', false, textToInsert)) { // Fallback to paste event if execCommand fails. @@ -49,13 +44,16 @@ async function submitQuote(page, text, tweetId) { await new Promise(r => setTimeout(r, 1000)); - // Confirm the quoted card is rendered before submitting; otherwise we may - // accidentally post a plain tweet without the quote attachment. + // Confirm the quoted card is rendered before submitting; otherwise + // we may accidentally post a plain tweet without the quote + // attachment. The compose page does not wrap the card in an + //
, so we probe the document for any link whose path + // exactly matches the requested status id (uses __twHasLinkToTarget + // from buildTwitterArticleScopeSource). let cardAttempts = 0; let hasQuoteCard = false; while (cardAttempts < 20) { - hasQuoteCard = Array.from(document.querySelectorAll('a[href*="/status/"]')) - .some((link) => getStatusId(link.href) === tweetId); + hasQuoteCard = __twHasLinkToTarget(document); if (hasQuoteCard) break; await new Promise(r => setTimeout(r, 250)); cardAttempts++; @@ -102,38 +100,68 @@ cli({ site: 'twitter', name: 'quote', access: 'write', - description: 'Quote-tweet a specific tweet with your own text', + description: 'Quote-tweet a specific tweet with your own text, optionally with a local or remote image', domain: 'x.com', strategy: Strategy.UI, browser: true, args: [ { name: 'url', type: 'string', required: true, positional: true, help: 'The URL of the tweet to quote' }, { name: 'text', type: 'string', required: true, positional: true, help: 'The text content of your quote' }, + { name: 'image', help: 'Optional local image path to attach to the quote tweet' }, + { name: 'image-url', help: 'Optional remote image URL to download and attach to the quote tweet' }, ], columns: ['status', 'message', 'text'], func: async (page, kwargs) => { if (!page) throw new CommandExecutionError('Browser session required for twitter quote'); + if (kwargs.image && kwargs['image-url']) { + throw new CommandExecutionError('Use either --image or --image-url, not both.'); + } - // Dedicated composer is more reliable than the inline quote-tweet button. + // Validate URL (typed ArgumentError on malformed/off-domain inputs) + // before any browser interaction or remote image download. const target = parseTweetUrl(kwargs.url); - await page.goto(`https://x.com/compose/post?url=${encodeURIComponent(target.url)}`, { waitUntil: 'load', settleMs: 2500 }); - await page.wait({ selector: '[data-testid="tweetTextarea_0"]', timeout: 15 }); - const result = await submitQuote(page, kwargs.text, target.id); - if (result.ok) { - // Wait for network submission to complete - await page.wait(3); + let localImagePath; + let cleanupDir; + try { + if (kwargs.image) { + localImagePath = resolveImagePath(kwargs.image); + } else if (kwargs['image-url']) { + const downloaded = await downloadRemoteImage(kwargs['image-url']); + localImagePath = downloaded.absPath; + cleanupDir = downloaded.cleanupDir; + } + + // Dedicated composer is more reliable than the inline quote-tweet button. + await page.goto(`https://x.com/compose/post?url=${encodeURIComponent(target.url)}`, { waitUntil: 'load', settleMs: 2500 }); + await page.wait({ selector: '[data-testid="tweetTextarea_0"]', timeout: 15 }); + + if (localImagePath) { + await page.wait({ selector: COMPOSER_FILE_INPUT_SELECTOR, timeout: 20 }); + await attachComposerImage(page, localImagePath); + } + + const result = await submitQuote(page, kwargs.text, target.id); + if (result.ok) { + // Wait for network submission to complete + await page.wait(3); + } + return [{ + status: result.ok ? 'success' : 'failed', + message: result.message, + text: kwargs.text, + ...(kwargs.image ? { image: kwargs.image } : {}), + ...(kwargs['image-url'] ? { 'image-url': kwargs['image-url'] } : {}), + }]; + } finally { + if (cleanupDir) { + fs.rmSync(cleanupDir, { recursive: true, force: true }); + } } - return [{ - status: result.ok ? 'success' : 'failed', - message: result.message, - text: kwargs.text, - }]; } }); export const __test__ = { buildQuoteComposerUrl, - extractTweetId, }; diff --git a/clis/twitter/quote.test.js b/clis/twitter/quote.test.js index 29434777b..7c92bb76b 100644 --- a/clis/twitter/quote.test.js +++ b/clis/twitter/quote.test.js @@ -1,4 +1,7 @@ -import { describe, expect, it } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; import { ArgumentError, CommandExecutionError } from '@jackwener/opencli/errors'; import { getRegistry } from '@jackwener/opencli/registry'; import { __test__ } from './quote.js'; @@ -6,11 +9,6 @@ import './quote.js'; import { createPageMock } from '../test-utils.js'; describe('twitter quote helpers', () => { - it('extracts tweet ids from both user and i/status URLs', () => { - expect(__test__.extractTweetId('https://x.com/alice/status/2040254679301718161?s=20')).toBe('2040254679301718161'); - expect(__test__.extractTweetId('https://x.com/i/status/2040318731105313143')).toBe('2040318731105313143'); - }); - it('builds the quote composer URL with the source tweet attached as ?url=...', () => { const composeUrl = __test__.buildQuoteComposerUrl('https://x.com/alice/status/2040254679301718161?s=20'); // The full source URL is round-tripped via encodeURIComponent — decoding it @@ -48,11 +46,14 @@ describe('twitter quote command', () => { const script = page.evaluate.mock.calls[0][0]; // Quote-attachment guard: the script must verify the quoted card rendered // before submitting; otherwise we'd silently post a plain tweet without - // the quote attachment. + // the quote attachment. Detection now uses the shared helper's + // __twHasLinkToTarget(document) — JSDOM coverage in shared.test.js + // proves it does an exact (not substring) match on the status id. expect(script).toContain('Quote target did not render'); expect(script).toContain('document.execCommand'); expect(script).toContain('tweetButton'); - expect(script).toContain('getStatusId(link.href) === tweetId'); + expect(script).toContain('__twHasLinkToTarget(document)'); + expect(script).toContain('__twGetStatusIdFromHref'); expect(script).toContain('Quote tweet submission did not complete before timeout'); expect(script).toContain('[role="alert"], [data-testid="toast"]'); expect(result).toEqual([ @@ -64,6 +65,93 @@ describe('twitter quote command', () => { ]); }); + it('uploads a local image through the quote composer when --image is provided', async () => { + const cmd = getRegistry().get('twitter/quote'); + expect(cmd?.func).toBeTypeOf('function'); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'opencli-twitter-quote-')); + const imagePath = path.join(tempDir, 'banner.png'); + fs.writeFileSync(imagePath, Buffer.from([0x89, 0x50, 0x4e, 0x47])); + const setFileInput = vi.fn().mockResolvedValue(undefined); + const page = createPageMock([ + { ok: true, previewCount: 1 }, + { ok: true, message: 'Quote tweet posted successfully.' }, + ], { + setFileInput, + }); + const result = await cmd.func(page, { + url: 'https://x.com/alice/status/2040254679301718161', + text: 'check this', + image: imagePath, + }); + expect(page.goto).toHaveBeenCalledWith( + 'https://x.com/compose/post?url=https%3A%2F%2Fx.com%2Falice%2Fstatus%2F2040254679301718161', + { waitUntil: 'load', settleMs: 2500 }, + ); + expect(page.wait).toHaveBeenNthCalledWith(1, { selector: '[data-testid="tweetTextarea_0"]', timeout: 15 }); + expect(page.wait).toHaveBeenNthCalledWith(2, { selector: 'input[type="file"][data-testid="fileInput"]', timeout: 20 }); + expect(setFileInput).toHaveBeenCalledWith([imagePath], 'input[type="file"][data-testid="fileInput"]'); + expect(result).toEqual([ + { + status: 'success', + message: 'Quote tweet posted successfully.', + text: 'check this', + image: imagePath, + }, + ]); + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + it('downloads a remote image before uploading when --image-url is provided', async () => { + const cmd = getRegistry().get('twitter/quote'); + expect(cmd?.func).toBeTypeOf('function'); + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + headers: { + get: vi.fn().mockReturnValue('image/png'), + }, + arrayBuffer: vi.fn().mockResolvedValue(Uint8Array.from([0x89, 0x50, 0x4e, 0x47]).buffer), + }); + vi.stubGlobal('fetch', fetchMock); + const setFileInput = vi.fn().mockResolvedValue(undefined); + const page = createPageMock([ + { ok: true, previewCount: 1 }, + { ok: true, message: 'Quote tweet posted successfully.' }, + ], { + setFileInput, + }); + const result = await cmd.func(page, { + url: 'https://x.com/alice/status/2040254679301718161', + text: 'remote attach', + 'image-url': 'https://example.com/banner', + }); + expect(fetchMock).toHaveBeenCalledWith('https://example.com/banner'); + expect(setFileInput).toHaveBeenCalledTimes(1); + const uploadedPath = setFileInput.mock.calls[0][0][0]; + expect(uploadedPath).toMatch(/opencli-twitter-.*\/image\.png$/); + // Per-call tmp dir is removed in the adapter's finally block. + expect(fs.existsSync(uploadedPath)).toBe(false); + expect(result).toEqual([ + { + status: 'success', + message: 'Quote tweet posted successfully.', + text: 'remote attach', + 'image-url': 'https://example.com/banner', + }, + ]); + vi.unstubAllGlobals(); + }); + + it('rejects using --image and --image-url together', async () => { + const cmd = getRegistry().get('twitter/quote'); + const page = createPageMock([]); + await expect(cmd.func(page, { + url: 'https://x.com/alice/status/2040254679301718161', + text: 'nope', + image: '/tmp/a.png', + 'image-url': 'https://example.com/a.png', + })).rejects.toThrow(CommandExecutionError); + }); + it('returns a failed row when the quote target fails to render', async () => { const cmd = getRegistry().get('twitter/quote'); expect(cmd?.func).toBeTypeOf('function'); diff --git a/clis/twitter/reply.js b/clis/twitter/reply.js index 8990cefa8..1ed441099 100644 --- a/clis/twitter/reply.js +++ b/clis/twitter/reply.js @@ -1,174 +1,24 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; -import * as os from 'node:os'; import { CommandExecutionError } from '@jackwener/opencli/errors'; import { cli, Strategy } from '@jackwener/opencli/registry'; -const REPLY_FILE_INPUT_SELECTOR = 'input[type="file"][data-testid="fileInput"]'; -const SUPPORTED_IMAGE_EXTENSIONS = new Set([ - '.jpg', - '.jpeg', - '.png', - '.gif', - '.webp', -]); -const MAX_IMAGE_SIZE_BYTES = 20 * 1024 * 1024; // 20 MB (Twitter allows 5MB images, 15MB GIFs) -const CONTENT_TYPE_TO_EXTENSION = { - 'image/jpeg': '.jpg', - 'image/jpg': '.jpg', - 'image/png': '.png', - 'image/gif': '.gif', - 'image/webp': '.webp', -}; -function resolveImagePath(imagePath) { - const absPath = path.resolve(imagePath); - if (!fs.existsSync(absPath)) { - throw new Error(`Image file not found: ${absPath}`); - } - const ext = path.extname(absPath).toLowerCase(); - if (!SUPPORTED_IMAGE_EXTENSIONS.has(ext)) { - throw new Error(`Unsupported image format "${ext}". Supported: jpg, jpeg, png, gif, webp`); - } - const stat = fs.statSync(absPath); - if (stat.size > MAX_IMAGE_SIZE_BYTES) { - throw new Error(`Image too large: ${(stat.size / 1024 / 1024).toFixed(1)} MB (max ${MAX_IMAGE_SIZE_BYTES / 1024 / 1024} MB)`); - } - return absPath; -} -function extractTweetId(url) { - let pathname = ''; - try { - pathname = new URL(url).pathname; - } - catch { - throw new Error(`Invalid tweet URL: ${url}`); - } - const match = pathname.match(/\/status\/(\d+)/); - if (!match?.[1]) { - throw new Error(`Could not extract tweet ID from URL: ${url}`); - } - return match[1]; -} -function buildReplyComposerUrl(url) { - return `https://x.com/compose/post?in_reply_to=${extractTweetId(url)}`; -} -function resolveImageExtension(url, contentType) { - const normalizedContentType = (contentType || '').split(';')[0].trim().toLowerCase(); - if (normalizedContentType && CONTENT_TYPE_TO_EXTENSION[normalizedContentType]) { - return CONTENT_TYPE_TO_EXTENSION[normalizedContentType]; - } - try { - const pathname = new URL(url).pathname; - const ext = path.extname(pathname).toLowerCase(); - if (SUPPORTED_IMAGE_EXTENSIONS.has(ext)) - return ext; - } - catch { - // Fall through to the final error below. - } - throw new Error(`Unsupported remote image format "${normalizedContentType || 'unknown'}". ` + - 'Supported: jpg, jpeg, png, gif, webp'); -} -async function downloadRemoteImage(imageUrl) { - let parsed; - try { - parsed = new URL(imageUrl); - } - catch { - throw new Error(`Invalid image URL: ${imageUrl}`); - } - if (!/^https?:$/.test(parsed.protocol)) { - throw new Error(`Unsupported image URL protocol: ${parsed.protocol}`); - } - const response = await fetch(imageUrl); - if (!response.ok) { - throw new Error(`Image download failed: HTTP ${response.status}`); - } - const contentLength = Number(response.headers.get('content-length') || '0'); - if (contentLength > MAX_IMAGE_SIZE_BYTES) { - throw new Error(`Image too large: ${(contentLength / 1024 / 1024).toFixed(1)} MB (max ${MAX_IMAGE_SIZE_BYTES / 1024 / 1024} MB)`); - } - const ext = resolveImageExtension(imageUrl, response.headers.get('content-type')); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'opencli-twitter-reply-')); - const tmpPath = path.join(tmpDir, `image${ext}`); - const buffer = Buffer.from(await response.arrayBuffer()); - if (buffer.byteLength > MAX_IMAGE_SIZE_BYTES) { - fs.rmSync(tmpDir, { recursive: true, force: true }); - throw new Error(`Image too large: ${(buffer.byteLength / 1024 / 1024).toFixed(1)} MB (max ${MAX_IMAGE_SIZE_BYTES / 1024 / 1024} MB)`); - } - fs.writeFileSync(tmpPath, buffer); - return tmpPath; -} -async function attachReplyImage(page, absImagePath) { - let uploaded = false; - if (page.setFileInput) { - try { - await page.setFileInput([absImagePath], REPLY_FILE_INPUT_SELECTOR); - uploaded = true; - } - catch (err) { - const msg = err instanceof Error ? err.message : String(err); - if (!msg.includes('Unknown action') && !msg.includes('not supported')) { - throw new Error(`Image upload failed: ${msg}`); - } - // setFileInput not supported by extension — fall through to base64 fallback - } - } - if (!uploaded) { - const ext = path.extname(absImagePath).toLowerCase(); - const mimeType = ext === '.png' - ? 'image/png' - : ext === '.gif' - ? 'image/gif' - : ext === '.webp' - ? 'image/webp' - : 'image/jpeg'; - const base64 = fs.readFileSync(absImagePath).toString('base64'); - if (base64.length > 500_000) { - console.warn(`[warn] Image base64 payload is ${(base64.length / 1024 / 1024).toFixed(1)}MB. ` + - 'This may fail with the browser bridge. Update the extension to v1.6+ for CDP-based upload, ' + - 'or compress the image before attaching.'); - } - const upload = await page.evaluate(` - (() => { - const input = document.querySelector(${JSON.stringify(REPLY_FILE_INPUT_SELECTOR)}); - if (!input) return { ok: false, error: 'No file input found on page' }; - - const binary = atob(${JSON.stringify(base64)}); - const bytes = new Uint8Array(binary.length); - for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); - - const dt = new DataTransfer(); - const blob = new Blob([bytes], { type: ${JSON.stringify(mimeType)} }); - dt.items.add(new File([blob], ${JSON.stringify(path.basename(absImagePath))}, { type: ${JSON.stringify(mimeType)} })); +import { parseTweetUrl } from './shared.js'; +import { + COMPOSER_FILE_INPUT_SELECTOR, + attachComposerImage, + downloadRemoteImage, + resolveImagePath, +} from './utils.js'; - Object.defineProperty(input, 'files', { value: dt.files, writable: false }); - input.dispatchEvent(new Event('change', { bubbles: true })); - input.dispatchEvent(new Event('input', { bubbles: true })); - return { ok: true }; - })() - `); - if (!upload?.ok) { - throw new Error(`Image upload failed: ${upload?.error ?? 'unknown error'}`); - } - } - await page.wait(2); - const uploadState = await page.evaluate(` - (() => { - const previewCount = document.querySelectorAll( - '[data-testid="attachments"] img, [data-testid="attachments"] video, [data-testid="tweetPhoto"]' - ).length; - const hasMedia = previewCount > 0 - || !!document.querySelector('[data-testid="attachments"]') - || !!Array.from(document.querySelectorAll('button,[role="button"]')).find((el) => - /remove media|remove image|remove/i.test((el.getAttribute('aria-label') || '') + ' ' + (el.textContent || '')) - ); - return { ok: hasMedia, previewCount }; - })() - `); - if (!uploadState?.ok) { - throw new Error('Image upload failed: preview did not appear.'); - } +function buildReplyComposerUrl(rawUrl) { + // Replaces the legacy local extractTweetId which used `/\/status\/(\d+)/` + // (silent: matched `/status/1234567` on substring `/status/123` and + // accepted any host). parseTweetUrl bubbles ArgumentError on + // malformed/off-domain inputs. + const target = parseTweetUrl(rawUrl); + return `https://x.com/compose/post?in_reply_to=${target.id}`; } + async function submitReply(page, text) { return page.evaluate(`(async () => { try { @@ -210,6 +60,7 @@ async function submitReply(page, text) { } })()`); } + cli({ site: 'twitter', name: 'reply', @@ -229,24 +80,24 @@ cli({ if (!page) throw new CommandExecutionError('Browser session required for twitter reply'); if (kwargs.image && kwargs['image-url']) { - throw new Error('Use either --image or --image-url, not both.'); + throw new CommandExecutionError('Use either --image or --image-url, not both.'); } let localImagePath; let cleanupDir; try { if (kwargs.image) { localImagePath = resolveImagePath(kwargs.image); - } - else if (kwargs['image-url']) { - localImagePath = await downloadRemoteImage(kwargs['image-url']); - cleanupDir = path.dirname(localImagePath); + } else if (kwargs['image-url']) { + const downloaded = await downloadRemoteImage(kwargs['image-url']); + localImagePath = downloaded.absPath; + cleanupDir = downloaded.cleanupDir; } // Dedicated composer is more reliable than the inline tweet page reply box. await page.goto(buildReplyComposerUrl(kwargs.url), { waitUntil: 'load', settleMs: 2500 }); await page.wait({ selector: '[data-testid="tweetTextarea_0"]', timeout: 15 }); if (localImagePath) { - await page.wait({ selector: REPLY_FILE_INPUT_SELECTOR, timeout: 20 }); - await attachReplyImage(page, localImagePath); + await page.wait({ selector: COMPOSER_FILE_INPUT_SELECTOR, timeout: 20 }); + await attachComposerImage(page, localImagePath); } const result = await submitReply(page, kwargs.text); if (result.ok) { @@ -259,8 +110,7 @@ cli({ ...(kwargs.image ? { image: kwargs.image } : {}), ...(kwargs['image-url'] ? { 'image-url': kwargs['image-url'] } : {}), }]; - } - finally { + } finally { if (cleanupDir) { fs.rmSync(cleanupDir, { recursive: true, force: true }); } @@ -269,8 +119,4 @@ cli({ }); export const __test__ = { buildReplyComposerUrl, - downloadRemoteImage, - extractTweetId, - resolveImageExtension, - resolveImagePath, }; diff --git a/clis/twitter/reply.test.js b/clis/twitter/reply.test.js index 25641f995..9a7c4f03b 100644 --- a/clis/twitter/reply.test.js +++ b/clis/twitter/reply.test.js @@ -2,9 +2,12 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { describe, expect, it, vi } from 'vitest'; +import { ArgumentError, CommandExecutionError } from '@jackwener/opencli/errors'; import { getRegistry } from '@jackwener/opencli/registry'; import { __test__ } from './reply.js'; +import { __test__ as utilsTest } from './utils.js'; import { createPageMock } from '../test-utils.js'; + describe('twitter reply command', () => { it('uses the dedicated reply composer for text-only replies too', async () => { const cmd = getRegistry().get('twitter/reply'); @@ -83,7 +86,11 @@ describe('twitter reply command', () => { expect(fetchMock).toHaveBeenCalledWith('https://example.com/qr'); expect(setFileInput).toHaveBeenCalledTimes(1); const uploadedPath = setFileInput.mock.calls[0][0][0]; - expect(uploadedPath).toMatch(/opencli-twitter-reply-.*\/image\.png$/); + // Tmp dir is created by utils.downloadRemoteImage with the + // 'opencli-twitter-' prefix; final extension comes from Content-Type. + expect(uploadedPath).toMatch(/opencli-twitter-.*\/image\.png$/); + // Per-call tmp dir is removed in the adapter's finally block, so the + // downloaded file no longer exists once the command returns. expect(fs.existsSync(uploadedPath)).toBe(false); expect(result).toEqual([ { @@ -95,10 +102,6 @@ describe('twitter reply command', () => { ]); vi.unstubAllGlobals(); }); - it('rejects invalid image paths early', async () => { - await expect(() => __test__.resolveImagePath('/tmp/does-not-exist.png')) - .toThrow('Image file not found'); - }); it('rejects using --image and --image-url together', async () => { const cmd = getRegistry().get('twitter/reply'); expect(cmd?.func).toBeTypeOf('function'); @@ -108,16 +111,31 @@ describe('twitter reply command', () => { text: 'nope', image: '/tmp/a.png', 'image-url': 'https://example.com/a.png', - })).rejects.toThrow('Use either --image or --image-url, not both.'); + })).rejects.toThrow(CommandExecutionError); + }); + it('rejects malformed tweet URLs before any browser interaction', () => { + // buildReplyComposerUrl runs parseTweetUrl synchronously; substring matches + // and off-domain hosts now throw ArgumentError instead of silently + // producing a wrong-host /compose/post URL. + expect(() => __test__.buildReplyComposerUrl('https://x.com/alice/home')).toThrow(ArgumentError); + expect(() => __test__.buildReplyComposerUrl('https://x.com.evil.com/alice/status/2040254679301718161')).toThrow(ArgumentError); + expect(() => __test__.buildReplyComposerUrl('not a url')).toThrow(ArgumentError); }); - it('extracts tweet ids from both user and i/status URLs', () => { - expect(__test__.extractTweetId('https://x.com/_kop6/status/2040254679301718161?s=20')).toBe('2040254679301718161'); - expect(__test__.extractTweetId('https://x.com/i/status/2040318731105313143')).toBe('2040318731105313143'); + it('builds the reply composer URL for both //status/ and /i/status/ shapes', () => { + expect(__test__.buildReplyComposerUrl('https://x.com/_kop6/status/2040254679301718161?s=20')) + .toBe('https://x.com/compose/post?in_reply_to=2040254679301718161'); expect(__test__.buildReplyComposerUrl('https://x.com/i/status/2040318731105313143')) .toBe('https://x.com/compose/post?in_reply_to=2040318731105313143'); }); +}); + +describe('twitter image helpers (utils.js)', () => { + it('rejects invalid image paths early', () => { + expect(() => utilsTest.resolveImagePath('/tmp/does-not-exist.png')) + .toThrow(ArgumentError); + }); it('prefers content-type when resolving remote image extensions', () => { - expect(__test__.resolveImageExtension('https://example.com/no-ext', 'image/webp')).toBe('.webp'); - expect(__test__.resolveImageExtension('https://example.com/a.jpeg?x=1', null)).toBe('.jpeg'); + expect(utilsTest.resolveImageExtension('https://example.com/no-ext', 'image/webp')).toBe('.webp'); + expect(utilsTest.resolveImageExtension('https://example.com/a.jpeg?x=1', null)).toBe('.jpeg'); }); }); diff --git a/clis/twitter/retweet.js b/clis/twitter/retweet.js index 3284eab0b..2ad0ea258 100644 --- a/clis/twitter/retweet.js +++ b/clis/twitter/retweet.js @@ -1,6 +1,6 @@ import { CommandExecutionError } from '@jackwener/opencli/errors'; import { cli, Strategy } from '@jackwener/opencli/registry'; -import { parseTweetUrl } from './shared.js'; +import { parseTweetUrl, buildTwitterArticleScopeSource } from './shared.js'; cli({ site: 'twitter', @@ -22,18 +22,11 @@ cli({ await page.wait({ selector: '[data-testid="primaryColumn"]' }); const result = await page.evaluate(`(async () => { try { - const tweetId = ${JSON.stringify(target.id)}; - const findTargetArticle = () => Array.from(document.querySelectorAll('article')).find((article) => - Array.from(article.querySelectorAll('a[href*="/status/"]')).some((link) => { - try { - const match = new URL(link.href, window.location.origin).pathname.match(/^\/(?:[^/]+|i)\/status\/(\d+)\/?$/); - return match?.[1] === tweetId; - } catch { - return false; - } - }) - ); - // Poll for the tweet to render + ${buildTwitterArticleScopeSource(target.id)} + // Poll for the tweet to render. State probes scoped to the article + // matching the requested status id — bare querySelector on a + // conversation page would silently grab the first article (e.g. + // the parent tweet) and retweet the wrong one. let attempts = 0; let retweetBtn = null; let unretweetBtn = null; @@ -62,7 +55,9 @@ cli({ // Step 1: click Retweet button → opens menu retweetBtn.click(); - // Step 2: wait for the confirm menu item to appear, then click it + // Step 2: wait for and click the confirm menu item. The confirm + // popover renders at the document root, not inside the article, + // so this lookup is intentionally document-scoped. let confirmBtn = null; for (let i = 0; i < 20; i++) { await new Promise(r => setTimeout(r, 250)); diff --git a/clis/twitter/retweet.test.js b/clis/twitter/retweet.test.js index 74b66eda2..d1810c138 100644 --- a/clis/twitter/retweet.test.js +++ b/clis/twitter/retweet.test.js @@ -24,8 +24,12 @@ describe('twitter retweet command', () => { expect(script).toContain('retweetBtn.click()'); expect(script).toContain("document.querySelector('[data-testid=\"retweetConfirm\"]')"); expect(script).toContain('confirmBtn.click()'); + // Article scoping comes from the shared helper (buildTwitterArticleScopeSource): + // emits __twHasLinkToTarget + __twGetStatusIdFromHref + the anchored + // tweet-path regex. JSDOM-level coverage lives in shared.test.js. + expect(script).toContain('__twHasLinkToTarget'); + expect(script).toContain('__twGetStatusIdFromHref'); expect(script).toContain("document.querySelectorAll('article')"); - expect(script).toContain('match?.[1] === tweetId'); expect(script).toContain("targetArticle?.querySelector('[data-testid=\"retweet\"]')"); // Idempotency probe: when already retweeted ([data-testid="unretweet"] present), // the script returns ok:true with an "already retweeted" message. diff --git a/clis/twitter/search.js b/clis/twitter/search.js index b63b1407b..193167b25 100644 --- a/clis/twitter/search.js +++ b/clis/twitter/search.js @@ -1,6 +1,104 @@ -import { CommandExecutionError } from '@jackwener/opencli/errors'; +import { ArgumentError, CommandExecutionError } from '@jackwener/opencli/errors'; import { cli, Strategy } from '@jackwener/opencli/registry'; import { extractMedia } from './shared.js'; +import { applyTopByEngagement } from './utils.js'; + +// ── Public-search operator surface ───────────────────────────────────── +// +// X's web search supports a small set of inline operators (from:, filter:, +// -filter:, etc.) plus a tab-selector URL param `f=`. We expose the most +// useful subset as flags so callers don't have to memorise the operator +// strings, while still letting power users append raw operators in . + +/** Operands accepted by `--has`. Map 1:1 to Twitter's `filter:` operator. */ +const HAS_CHOICES = Object.freeze(['media', 'images', 'videos', 'links', 'replies']); + +/** + * Operands accepted by `--exclude`. Note that `retweets` is exposed as the + * friendlier name but X's actual operator stays as `-filter:nativeretweets` + * (the historical "native" prefix is preserved by their backend). + */ +const EXCLUDE_CHOICES = Object.freeze(['replies', 'retweets', 'media', 'links']); + +/** + * Operands accepted by `--product`. `photos`/`videos` are the human-friendly + * forms used by the X UI tabs; the URL param uses the singular forms (image, + * video). `people` is intentionally NOT supported here because that tab + * returns User objects, not tweets, and would need a different output schema. + */ +const PRODUCT_CHOICES = Object.freeze(['top', 'live', 'photos', 'videos']); + +const PRODUCT_TO_F_PARAM = Object.freeze({ + top: 'top', + live: 'live', + photos: 'image', + videos: 'video', +}); + +const FROM_USER_PATTERN = /^[A-Za-z0-9_]{1,15}$/; + +const EXCLUDE_TO_OPERATOR = Object.freeze({ + replies: '-filter:replies', + // `retweets` is a CLI-friendly alias for X's actual `-filter:nativeretweets`. + retweets: '-filter:nativeretweets', + media: '-filter:media', + links: '-filter:links', +}); + +/** + * Compose the final search query string by appending operator clauses for + * --from / --has / --exclude. Pure synchronous — exported via __test__ for + * unit coverage. + * + * Behaviour notes: + * - Trims leading `@` from --from so callers can pass `@alice` or `alice`. + * - Order is ` from:X filter:Y -filter:Z` (matches what X's own search + * bar emits when you click the suggestions UI). + * - Empty with non-empty filters is allowed — the resulting string + * is just the operator clauses joined; X handles that fine. + * + * @param {string} rawQuery + * @param {{ from?: string, has?: string, exclude?: string }} kwargs + * @returns {string} + */ +function buildSearchQuery(rawQuery, kwargs) { + const parts = [String(rawQuery ?? '').trim()]; + if (kwargs.from) { + const fromUser = String(kwargs.from).trim().replace(/^@+/, ''); + if (fromUser && !FROM_USER_PATTERN.test(fromUser)) { + throw new ArgumentError( + `Invalid --from username: ${JSON.stringify(kwargs.from)}`, + 'Use a Twitter/X handle with 1-15 letters, numbers, or underscores; omit @ or pass @handle.', + ); + } + if (fromUser) parts.push(`from:${fromUser}`); + } + if (kwargs.has) { + parts.push(`filter:${kwargs.has}`); + } + if (kwargs.exclude) { + const op = EXCLUDE_TO_OPERATOR[kwargs.exclude]; + if (op) parts.push(op); + } + return parts.filter(Boolean).join(' '); +} + +/** + * Resolve which X search tab (`f=` URL param) to land on. `--product` wins + * over the legacy `--filter` so adding `--product` doesn't break callers that + * were already setting `--filter top|live`. + * + * @param {{ product?: string, filter?: string }} kwargs + * @returns {string} URL `f=` value: top|live|image|video + */ +function resolveSearchFParam(kwargs) { + if (kwargs.product) { + const mapped = PRODUCT_TO_F_PARAM[kwargs.product]; + if (mapped) return mapped; + } + return kwargs.filter === 'live' ? 'live' : 'top'; +} + /** * Trigger Twitter search SPA navigation with fallback strategies. * @@ -9,9 +107,13 @@ import { extractMedia } from './shared.js'; * intermittently (e.g. due to Twitter A/B tests or timing races — see #690). * * Both strategies preserve the JS context so the fetch interceptor stays alive. + * + * @param {object} page + * @param {string} query — final composed query (already merged with operators) + * @param {string} fParam — Twitter URL `f=` value (top|live|image|video) */ -async function navigateToSearch(page, query, filter) { - const searchUrl = JSON.stringify(`/search?q=${encodeURIComponent(query)}&f=${filter}`); +async function navigateToSearch(page, query, fParam) { + const searchUrl = JSON.stringify(`/search?q=${encodeURIComponent(query)}&f=${fParam}`); let lastPath = ''; // Strategy 1 (primary): pushState + popstate with retry for (let attempt = 1; attempt <= 2; attempt++) { @@ -74,40 +176,78 @@ async function navigateToSearch(page, query, filter) { } lastPath = String(await page.evaluate('() => window.location.pathname') || ''); if (lastPath.startsWith('/search')) { - if (filter === 'live') { - await page.evaluate(`(() => { - const tabs = document.querySelectorAll('[role="tab"]'); - for (const tab of tabs) { - if (tab.textContent.includes('Latest') || tab.textContent.includes('最新')) { - tab.click(); - return; - } - } - })()`); - await page.wait(2); + // The fallback path doesn't carry the f= URL param, so click the + // matching tab to align with the requested product. Only `live` + // currently surfaces a distinct tab label — `image`/`video` tabs + // also need an explicit click, so try them all. + const tabClicked = await clickProductTabIfNeeded(page, fParam); + if (!tabClicked) { + throw new CommandExecutionError(`SPA fallback reached /search but could not select the requested product tab: ${fParam}`); } return; } } throw new CommandExecutionError(`SPA navigation to /search failed. Final path: ${lastPath || '(empty)'}. Twitter may have changed its routing.`); } + +/** + * After the search-input fallback lands on /search, the f= param is missing + * from the URL. Click the matching tab in the result page header so the + * SearchTimeline call uses the right filter. No-op for fParam=top (default). + */ +async function clickProductTabIfNeeded(page, fParam) { + if (fParam === 'top') return true; + const tabLabels = JSON.stringify({ + live: ['Latest', '最新'], + image: ['Photos', 'Images', '照片', '图片'], + video: ['Videos', '视频'], + }[fParam] || []); + if (tabLabels === '[]') return true; + const clicked = await page.evaluate(`(() => { + const labels = ${tabLabels}; + const tabs = document.querySelectorAll('[role="tab"]'); + for (const tab of tabs) { + const txt = (tab.textContent || '').trim(); + if (labels.some(l => txt.includes(l))) { + tab.click(); + return true; + } + } + return false; + })()`); + if (!clicked) return false; + await page.wait(2); + return true; +} + cli({ site: 'twitter', name: 'search', access: 'read', - description: 'Search Twitter/X for tweets', + description: 'Search Twitter/X for tweets, with optional --from / --has / --exclude / --product filters mapped to X\'s search operators', domain: 'x.com', strategy: Strategy.INTERCEPT, // Use intercept strategy browser: true, args: [ - { name: 'query', type: 'string', required: true, positional: true, help: 'Twitter/X search query (operators like `from:` and `since:` are supported)' }, - { name: 'filter', type: 'string', default: 'top', choices: ['top', 'live'] }, - { name: 'limit', type: 'int', default: 15 }, + { name: 'query', type: 'string', required: true, positional: true, help: 'Search query. Raw X operators (e.g. "exact phrase", #tag, OR, lang:en, since:YYYY-MM-DD, from:, since:) are passed through unchanged.' }, + { name: 'filter', type: 'string', default: 'top', choices: ['top', 'live'], help: 'Legacy alias for --product. Kept for backwards compatibility; if --product is set it wins.' }, + { name: 'product', type: 'string', choices: PRODUCT_CHOICES, help: 'Which X search tab to read: top (default), live (Latest), photos, videos. Maps to the f= URL param.' }, + { name: 'from', type: 'string', help: 'Restrict to tweets authored by . Leading @ is stripped. Equivalent to appending `from:` to the query.' }, + { name: 'has', type: 'string', choices: HAS_CHOICES, help: 'Restrict to tweets that have media|images|videos|links|replies. Maps to X\'s `filter:` operator.' }, + { name: 'exclude', type: 'string', choices: EXCLUDE_CHOICES, help: 'Exclude tweets matching : replies|retweets|media|links. Maps to X\'s `-filter:` operator (retweets → -filter:nativeretweets).' }, + { name: 'limit', type: 'int', default: 15, help: 'Maximum number of tweets to return (default 15). Result count after server-side filtering.' }, + { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the results by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps X\'s native ordering.' }, ], columns: ['id', 'author', 'text', 'created_at', 'likes', 'views', 'url', 'has_media', 'media_urls'], func: async (page, kwargs) => { - const query = kwargs.query; - const filter = kwargs.filter === 'live' ? 'live' : 'top'; + const finalQuery = buildSearchQuery(kwargs.query, kwargs); + if (!finalQuery) { + throw new ArgumentError('twitter search query is empty', 'Provide a non-empty , or use at least one of --from / --has / --exclude.'); + } + if (!Number.isInteger(Number(kwargs.limit)) || Number(kwargs.limit) <= 0) { + throw new ArgumentError('twitter search --limit must be a positive integer', 'Example: opencli twitter search opencli --limit 15'); + } + const fParam = resolveSearchFParam(kwargs); // 1. Navigate to x.com/explore (has a search input at the top) await page.goto('https://x.com/explore'); await page.wait(3); @@ -120,10 +260,10 @@ cli({ // a full page reload, so the interceptor stays alive. // Note: the previous approach (nativeSetter + Enter keydown on the // search input) does not reliably trigger Twitter's form submission. - await navigateToSearch(page, query, filter); + await navigateToSearch(page, finalQuery, fParam); // 4. Scroll to trigger additional pagination await page.autoScroll({ times: 3, delayMs: 2000 }); - // 6. Retrieve captured data + // 5. Retrieve captured data const requests = await page.getInterceptedRequests(); if (!requests || requests.length === 0) return []; @@ -167,6 +307,18 @@ cli({ // ignore parsing errors for individual payloads } } - return results.slice(0, kwargs.limit); + const trimmed = results.slice(0, kwargs.limit); + return applyTopByEngagement(trimmed, kwargs['top-by-engagement']); } }); + +export const __test__ = { + buildSearchQuery, + resolveSearchFParam, + HAS_CHOICES, + EXCLUDE_CHOICES, + PRODUCT_CHOICES, + EXCLUDE_TO_OPERATOR, + PRODUCT_TO_F_PARAM, + FROM_USER_PATTERN, +}; diff --git a/clis/twitter/search.test.js b/clis/twitter/search.test.js index 8e24dc05e..3041c78e8 100644 --- a/clis/twitter/search.test.js +++ b/clis/twitter/search.test.js @@ -1,6 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; import { getRegistry } from '@jackwener/opencli/registry'; -import './search.js'; +import { __test__ } from './search.js'; + +const { buildSearchQuery, resolveSearchFParam, HAS_CHOICES, EXCLUDE_CHOICES, PRODUCT_CHOICES, EXCLUDE_TO_OPERATOR, PRODUCT_TO_F_PARAM, FROM_USER_PATTERN } = __test__; describe('twitter search command', () => { it('retries transient SPA navigation failures before giving up', async () => { const command = getRegistry().get('twitter/search'); @@ -213,6 +215,56 @@ describe('twitter search command', () => { expect(evaluate).toHaveBeenCalledTimes(6); expect(page.autoScroll).toHaveBeenCalled(); }); + it('clicks the requested product tab after fallback navigation when f= param is absent', async () => { + const command = getRegistry().get('twitter/search'); + expect(command?.func).toBeTypeOf('function'); + const evaluate = vi.fn() + .mockResolvedValueOnce(undefined) // pushState attempt 1 + .mockResolvedValueOnce('/explore') + .mockResolvedValueOnce(undefined) // pushState attempt 2 + .mockResolvedValueOnce('/explore') + .mockResolvedValueOnce({ ok: true }) // search input fallback + .mockResolvedValueOnce('/search') + .mockResolvedValueOnce(true); // product tab click + const page = { + goto: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + installInterceptor: vi.fn().mockResolvedValue(undefined), + evaluate, + autoScroll: vi.fn().mockResolvedValue(undefined), + getInterceptedRequests: vi.fn().mockResolvedValue([]), + }; + const result = await command.func(page, { query: 'cats', product: 'photos', limit: 5 }); + expect(result).toEqual([]); + expect(evaluate).toHaveBeenCalledTimes(7); + expect(evaluate.mock.calls[6][0]).toContain('Photos'); + expect(page.autoScroll).toHaveBeenCalled(); + }); + it('throws when fallback navigation cannot select the requested product tab', async () => { + const command = getRegistry().get('twitter/search'); + expect(command?.func).toBeTypeOf('function'); + const evaluate = vi.fn() + .mockResolvedValueOnce(undefined) // pushState attempt 1 + .mockResolvedValueOnce('/explore') + .mockResolvedValueOnce(undefined) // pushState attempt 2 + .mockResolvedValueOnce('/explore') + .mockResolvedValueOnce({ ok: true }) // search input fallback + .mockResolvedValueOnce('/search') + .mockResolvedValueOnce(false); // requested tab missing + const page = { + goto: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + installInterceptor: vi.fn().mockResolvedValue(undefined), + evaluate, + autoScroll: vi.fn().mockResolvedValue(undefined), + getInterceptedRequests: vi.fn(), + }; + await expect(command.func(page, { query: 'cats', product: 'videos', limit: 5 })) + .rejects + .toThrow(/could not select the requested product tab: video/); + expect(page.autoScroll).not.toHaveBeenCalled(); + expect(page.getInterceptedRequests).not.toHaveBeenCalled(); + }); it('throws with the final path after both attempts fail', async () => { const command = getRegistry().get('twitter/search'); expect(command?.func).toBeTypeOf('function'); @@ -238,3 +290,216 @@ describe('twitter search command', () => { expect(evaluate).toHaveBeenCalledTimes(5); }); }); + +describe('twitter search filter helpers', () => { + describe('buildSearchQuery', () => { + it('returns the trimmed raw query when no filters are set', () => { + expect(buildSearchQuery(' hello world ', {})).toBe('hello world'); + }); + it('appends from: with leading @ stripped', () => { + expect(buildSearchQuery('hello', { from: '@alice' })).toBe('hello from:alice'); + }); + it('preserves from: when caller passes a bare username', () => { + expect(buildSearchQuery('hello', { from: 'alice' })).toBe('hello from:alice'); + }); + it('strips multiple leading @ characters from --from', () => { + expect(buildSearchQuery('hi', { from: '@@bob' })).toBe('hi from:bob'); + }); + it('drops --from when it is whitespace-only', () => { + expect(buildSearchQuery('hi', { from: ' ' })).toBe('hi'); + }); + it('rejects invalid --from usernames instead of injecting raw operators', () => { + expect(() => buildSearchQuery('hi', { from: 'alice filter:links' })).toThrow(/Invalid --from/); + expect(() => buildSearchQuery('hi', { from: 'alice/bob' })).toThrow(/Invalid --from/); + expect(() => buildSearchQuery('hi', { from: '@' + 'a'.repeat(16) })).toThrow(/Invalid --from/); + }); + it('appends filter: for --has', () => { + expect(buildSearchQuery('q', { has: 'images' })).toBe('q filter:images'); + }); + it('maps --exclude retweets to -filter:nativeretweets', () => { + expect(buildSearchQuery('q', { exclude: 'retweets' })).toBe('q -filter:nativeretweets'); + }); + it('maps --exclude replies/media/links to their -filter operators', () => { + expect(buildSearchQuery('q', { exclude: 'replies' })).toBe('q -filter:replies'); + expect(buildSearchQuery('q', { exclude: 'media' })).toBe('q -filter:media'); + expect(buildSearchQuery('q', { exclude: 'links' })).toBe('q -filter:links'); + }); + it('silently ignores unknown --exclude values', () => { + // choices: ['replies','retweets','media','links'] — unknowns shouldn't appear + // in real CLI use because the validator rejects them, but the helper still + // guards via the EXCLUDE_TO_OPERATOR map lookup. + expect(buildSearchQuery('q', { exclude: 'bogus' })).toBe('q'); + }); + it('composes multiple filter clauses in stable order: query → from → has → exclude', () => { + expect(buildSearchQuery('hot take', { + from: '@alice', + has: 'media', + exclude: 'retweets', + })).toBe('hot take from:alice filter:media -filter:nativeretweets'); + }); + it('allows an empty raw query when filters are present', () => { + expect(buildSearchQuery('', { from: 'alice' })).toBe('from:alice'); + }); + it('returns empty string when nothing useful is supplied', () => { + expect(buildSearchQuery('', {})).toBe(''); + expect(buildSearchQuery(' ', {})).toBe(''); + }); + it('coerces nullish raw query into empty string', () => { + expect(buildSearchQuery(null, { from: 'alice' })).toBe('from:alice'); + expect(buildSearchQuery(undefined, { from: 'alice' })).toBe('from:alice'); + }); + }); + + describe('resolveSearchFParam', () => { + it('defaults to top when neither product nor filter is set', () => { + expect(resolveSearchFParam({})).toBe('top'); + }); + it('returns top when filter=top', () => { + expect(resolveSearchFParam({ filter: 'top' })).toBe('top'); + }); + it('returns live when filter=live', () => { + expect(resolveSearchFParam({ filter: 'live' })).toBe('live'); + }); + it('maps --product photos to image (Twitter URL singular form)', () => { + expect(resolveSearchFParam({ product: 'photos' })).toBe('image'); + }); + it('maps --product videos to video (Twitter URL singular form)', () => { + expect(resolveSearchFParam({ product: 'videos' })).toBe('video'); + }); + it('maps --product top|live straight through', () => { + expect(resolveSearchFParam({ product: 'top' })).toBe('top'); + expect(resolveSearchFParam({ product: 'live' })).toBe('live'); + }); + it('lets --product win when both --product and --filter are set', () => { + expect(resolveSearchFParam({ product: 'photos', filter: 'live' })).toBe('image'); + expect(resolveSearchFParam({ product: 'top', filter: 'live' })).toBe('top'); + }); + it('falls back to filter when --product is unknown', () => { + // unknowns are blocked at the CLI validator layer; this is just defence + expect(resolveSearchFParam({ product: 'bogus', filter: 'live' })).toBe('live'); + expect(resolveSearchFParam({ product: 'bogus' })).toBe('top'); + }); + }); + + describe('choice surface', () => { + it('exposes the documented HAS_CHOICES set', () => { + expect(HAS_CHOICES).toEqual(['media', 'images', 'videos', 'links', 'replies']); + }); + it('exposes the documented EXCLUDE_CHOICES set', () => { + expect(EXCLUDE_CHOICES).toEqual(['replies', 'retweets', 'media', 'links']); + }); + it('exposes the documented PRODUCT_CHOICES set', () => { + expect(PRODUCT_CHOICES).toEqual(['top', 'live', 'photos', 'videos']); + }); + it('keeps PRODUCT_TO_F_PARAM domain a strict subset of PRODUCT_CHOICES', () => { + for (const choice of PRODUCT_CHOICES) { + expect(PRODUCT_TO_F_PARAM[choice]).toBeTypeOf('string'); + } + }); + it('keeps EXCLUDE_TO_OPERATOR domain a strict subset of EXCLUDE_CHOICES', () => { + for (const choice of EXCLUDE_CHOICES) { + expect(EXCLUDE_TO_OPERATOR[choice]).toMatch(/^-filter:/); + } + }); + it('keeps FROM_USER_PATTERN aligned with X handle syntax', () => { + expect(FROM_USER_PATTERN.test('alice_123')).toBe(true); + expect(FROM_USER_PATTERN.test('a'.repeat(15))).toBe(true); + expect(FROM_USER_PATTERN.test('a'.repeat(16))).toBe(false); + expect(FROM_USER_PATTERN.test('alice/bob')).toBe(false); + }); + }); +}); + +describe('twitter search end-to-end with new filters', () => { + it('encodes the composed query and product=live into the f= URL param', async () => { + const command = getRegistry().get('twitter/search'); + const evaluate = vi.fn() + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce('/search'); + const page = { + goto: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + installInterceptor: vi.fn().mockResolvedValue(undefined), + evaluate, + autoScroll: vi.fn().mockResolvedValue(undefined), + getInterceptedRequests: vi.fn().mockResolvedValue([]), + }; + await command.func(page, { + query: 'breaking news', + from: '@alice', + has: 'images', + exclude: 'retweets', + product: 'live', + limit: 5, + }); + const pushStateCall = evaluate.mock.calls[0][0]; + // f=live wins because --product=live trumps the default --filter + expect(pushStateCall).toContain('f=live'); + // composed query should be percent-encoded inside the URL + const encoded = encodeURIComponent('breaking news from:alice filter:images -filter:nativeretweets'); + expect(pushStateCall).toContain(encoded); + }); + it('throws ArgumentError when query and all filters are empty', async () => { + const command = getRegistry().get('twitter/search'); + const page = { + goto: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + installInterceptor: vi.fn().mockResolvedValue(undefined), + evaluate: vi.fn(), + autoScroll: vi.fn().mockResolvedValue(undefined), + getInterceptedRequests: vi.fn(), + }; + await expect(command.func(page, { query: ' ', limit: 5 })) + .rejects + .toThrow(/empty/i); + expect(page.installInterceptor).not.toHaveBeenCalled(); + }); + it('throws ArgumentError for invalid --from before navigation', async () => { + const command = getRegistry().get('twitter/search'); + const page = { + goto: vi.fn(), + wait: vi.fn(), + installInterceptor: vi.fn(), + evaluate: vi.fn(), + autoScroll: vi.fn(), + getInterceptedRequests: vi.fn(), + }; + await expect(command.func(page, { query: 'hi', from: 'alice filter:links', limit: 5 })) + .rejects + .toThrow(/Invalid --from/); + expect(page.goto).not.toHaveBeenCalled(); + }); + it('throws ArgumentError for invalid --limit before navigation', async () => { + const command = getRegistry().get('twitter/search'); + const page = { + goto: vi.fn(), + wait: vi.fn(), + installInterceptor: vi.fn(), + evaluate: vi.fn(), + autoScroll: vi.fn(), + getInterceptedRequests: vi.fn(), + }; + await expect(command.func(page, { query: 'hi', limit: 0 })) + .rejects + .toThrow(/--limit/); + expect(page.goto).not.toHaveBeenCalled(); + }); + it('runs with only filters set (empty )', async () => { + const command = getRegistry().get('twitter/search'); + const evaluate = vi.fn() + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce('/search'); + const page = { + goto: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + installInterceptor: vi.fn().mockResolvedValue(undefined), + evaluate, + autoScroll: vi.fn().mockResolvedValue(undefined), + getInterceptedRequests: vi.fn().mockResolvedValue([]), + }; + const result = await command.func(page, { query: '', from: 'alice', limit: 5 }); + expect(result).toEqual([]); + const pushStateCall = evaluate.mock.calls[0][0]; + expect(pushStateCall).toContain(encodeURIComponent('from:alice')); + }); +}); diff --git a/clis/twitter/shared.js b/clis/twitter/shared.js index 07ee5910f..414ce050d 100644 --- a/clis/twitter/shared.js +++ b/clis/twitter/shared.js @@ -36,6 +36,48 @@ export function parseTweetUrl(rawUrl) { }; } +/** + * Build a JS source fragment that, when embedded inside a `page.evaluate(...)` + * IIFE, declares browser-side helpers for scoping operations to a specific + * tweet by status id. Sibling adapters historically inlined ad-hoc article + * lookups that either (a) skipped scoping entirely (silent: act on first + * matching button on a conversation page) or (b) used substring matches like + * `pathname.includes('/status/' + tweetId)` (silent: `/status/123` matches + * `/status/1234567`). This helper centralises the canonical pattern so all + * write-actions reuse the same exact-match guard. + * + * Declared bindings (available to the embedding IIFE): + * - `tweetId` : the requested status id (string) + * - `__twGetStatusIdFromHref(href)` : extract status id from a link href, or null + * - `__twHasLinkToTarget(root)` : true iff `root` contains any link to tweetId + * - `findTargetArticle()` : the
matching tweetId, or undefined + */ +export function buildTwitterArticleScopeSource(tweetId) { + return ` + const tweetId = ${JSON.stringify(tweetId)}; + const __twTweetPathRe = /^\\/(?:[^/]+|i)\\/status\\/(\\d+)\\/?$/; + const __twIsTwitterHost = (hostname) => hostname === 'x.com' + || hostname === 'twitter.com' + || hostname.endsWith('.x.com') + || hostname.endsWith('.twitter.com'); + const __twGetStatusIdFromHref = (href) => { + try { + const parsed = new URL(href, window.location.origin); + if (parsed.protocol !== 'https:' || !__twIsTwitterHost(parsed.hostname.toLowerCase())) { + return null; + } + return parsed.pathname.match(__twTweetPathRe)?.[1] || null; + } catch { + return null; + } + }; + const __twHasLinkToTarget = (root) => Array.from(root.querySelectorAll('a[href*="/status/"]')) + .some((link) => __twGetStatusIdFromHref(link.href) === tweetId); + const findTargetArticle = () => Array.from(document.querySelectorAll('article')) + .find(__twHasLinkToTarget); + `; +} + export function sanitizeQueryId(resolved, fallbackId) { return typeof resolved === 'string' && QUERY_ID_PATTERN.test(resolved) ? resolved : fallbackId; } @@ -103,4 +145,5 @@ export const __test__ = { sanitizeQueryId, extractMedia, parseTweetUrl, + buildTwitterArticleScopeSource, }; diff --git a/clis/twitter/shared.test.js b/clis/twitter/shared.test.js index 0a21f904b..1ed8395ee 100644 --- a/clis/twitter/shared.test.js +++ b/clis/twitter/shared.test.js @@ -1,8 +1,9 @@ import { describe, expect, it } from 'vitest'; +import { JSDOM } from 'jsdom'; import { __test__ } from './shared.js'; import { ArgumentError } from '@jackwener/opencli/errors'; -const { extractMedia, parseTweetUrl } = __test__; +const { extractMedia, parseTweetUrl, buildTwitterArticleScopeSource } = __test__; describe('twitter parseTweetUrl', () => { it('accepts exact Twitter/X tweet URLs and preserves query parameters', () => { @@ -30,6 +31,111 @@ describe('twitter parseTweetUrl', () => { }); }); +describe('twitter buildTwitterArticleScopeSource', () => { + // JSDOM-based tests prove the returned source actually works on real DOM — + // mocked `evaluate` tests in adapter specs only verify the script string + // contains expected tokens, but cannot catch silent matching bugs (cf. + // dianping #1312: mocked-evaluate single tests miss in-browser logic bugs). + function loadHelpers(tweetId, dom) { + const source = buildTwitterArticleScopeSource(tweetId); + const probe = new Function( + 'document', + 'window', + 'URL', + `${source}\nreturn { findTargetArticle, __twHasLinkToTarget, __twGetStatusIdFromHref };`, + ); + return probe(dom.window.document, dom.window, dom.window.URL); + } + function makeDom(html) { + return new JSDOM(`${html}`, { url: 'https://x.com/alice/status/2040254679301718161' }); + } + + it('finds the article whose link exactly matches the requested status id', () => { + const dom = makeDom(` + + + `); + const helpers = loadHelpers('2040254679301718161', dom); + const article = helpers.findTargetArticle(); + expect(article?.id).toBe('a'); + }); + + it('rejects substring matches — tweet id 123 must not match /status/1234567', () => { + // This is the codex-mini0 #1400 catch (substring vulnerability): + // `/status/123` was accepted as a substring of `/status/1234567`. + const dom = makeDom(''); + const helpers = loadHelpers('123', dom); + expect(helpers.findTargetArticle()).toBeUndefined(); + }); + + it('rejects path-suffix attack — /status//photo/1 must not match status ', () => { + // Same regex anchor that parseTweetUrl uses — guards against attached + // paths like `/photo/1` that would otherwise pass with a loose suffix. + const dom = makeDom(''); + const helpers = loadHelpers('2040254679301718161', dom); + expect(helpers.findTargetArticle()).toBeUndefined(); + }); + + it('rejects off-domain links even when the path has the requested status id', () => { + const dom = makeDom(''); + const helpers = loadHelpers('2040254679301718161', dom); + expect(helpers.findTargetArticle()).toBeUndefined(); + }); + + it('rejects host-suffix and non-https status links', () => { + const dom = makeDom(` + + + `); + const helpers = loadHelpers('2040254679301718161', dom); + expect(helpers.findTargetArticle()).toBeUndefined(); + }); + + it('accepts exact Twitter/X status links with query and hash suffixes', () => { + const dom = makeDom(''); + const helpers = loadHelpers('2040254679301718161', dom); + expect(helpers.findTargetArticle()?.id).toBe('ok'); + }); + + it('matches /i/status/ URL form', () => { + const dom = makeDom(''); + const helpers = loadHelpers('2040318731105313143', dom); + expect(helpers.findTargetArticle()).toBeTruthy(); + }); + + it('__twHasLinkToTarget reports true on any descendant matching tweet id', () => { + // Used by quote-card guard in quote.js — the quoted tweet card is not + // inside an
, but somewhere on the compose page. + const dom = makeDom(` + + `); + const helpers = loadHelpers('2040254679301718161', dom); + expect(helpers.__twHasLinkToTarget(dom.window.document)).toBe(true); + }); + + it('__twGetStatusIdFromHref returns null on non-status URLs', () => { + const dom = makeDom(''); + const helpers = loadHelpers('123', dom); + expect(helpers.__twGetStatusIdFromHref('https://x.com/alice/home')).toBeNull(); + expect(helpers.__twGetStatusIdFromHref('https://x.com/alice/status/123/photo/1')).toBeNull(); + expect(helpers.__twGetStatusIdFromHref('https://evil.com/alice/status/123')).toBeNull(); + expect(helpers.__twGetStatusIdFromHref('https://x.com.evil.com/alice/status/123')).toBeNull(); + expect(helpers.__twGetStatusIdFromHref('http://x.com/alice/status/123')).toBeNull(); + expect(helpers.__twGetStatusIdFromHref('not a url')).toBeNull(); + }); + + it('emits the canonical regex anchor — guards future maintainers from dropping ^ or $', () => { + const source = buildTwitterArticleScopeSource('123'); + // Source-level assertion complements the JSDOM behavioural tests above. + // If a future refactor relaxes the anchor (e.g. drops ^ or $), the + // JSDOM tests would still pass on benign inputs but fail on adversarial + // cases. This token check ensures the regex shape itself is preserved. + expect(source).toContain('/^\\/(?:[^/]+|i)\\/status\\/(\\d+)\\/?$/'); + }); +}); + describe('twitter extractMedia', () => { it('returns false + empty list when legacy has no media', () => { expect(extractMedia({})).toEqual({ has_media: false, media_urls: [] }); diff --git a/clis/twitter/thread.js b/clis/twitter/thread.js index 9aaa36c88..e939c57f9 100644 --- a/clis/twitter/thread.js +++ b/clis/twitter/thread.js @@ -1,8 +1,8 @@ import { cli, Strategy } from '@jackwener/opencli/registry'; import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; import { extractMedia } from './shared.js'; +import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js'; // ── Twitter GraphQL constants ────────────────────────────────────────── -const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; const TWEET_DETAIL_QUERY_ID = 'nBS-WpgA6ZG0CyNHD517JQ'; const FEATURES = { responsive_web_graphql_exclude_directive_enabled: true, @@ -103,6 +103,7 @@ cli({ args: [ { name: 'tweet-id', positional: true, type: 'string', required: true, help: 'Tweet numeric ID (e.g. 1234567890) or full status URL' }, { name: 'limit', type: 'int', default: 50 }, + { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the thread by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the conversation\'s structural ordering.' }, ], columns: ['id', 'author', 'text', 'likes', 'retweets', 'url', 'has_media', 'media_urls'], func: async (page, kwargs) => { @@ -121,7 +122,7 @@ cli({ throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)'); // Build auth headers in TypeScript const headers = JSON.stringify({ - 'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`, + 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, 'X-Csrf-Token': ct0, 'X-Twitter-Auth-Type': 'OAuth2Session', 'X-Twitter-Active-User': 'yes', @@ -149,6 +150,7 @@ cli({ break; cursor = nextCursor; } - return allTweets.slice(0, kwargs.limit); + const trimmed = allTweets.slice(0, kwargs.limit); + return applyTopByEngagement(trimmed, kwargs['top-by-engagement']); }, }); diff --git a/clis/twitter/timeline.js b/clis/twitter/timeline.js index 4611b9b71..46bffa7c3 100644 --- a/clis/twitter/timeline.js +++ b/clis/twitter/timeline.js @@ -1,8 +1,8 @@ import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; import { cli, Strategy } from '@jackwener/opencli/registry'; import { resolveTwitterQueryId, extractMedia } from './shared.js'; +import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js'; // ── Twitter GraphQL constants ────────────────────────────────────────── -const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; const HOME_TIMELINE_QUERY_ID = 'c-CzHF1LboFilMpsx4ZCrQ'; const HOME_LATEST_TIMELINE_QUERY_ID = 'BKB7oi212Fi7kQtCBGE4zA'; // Endpoint config: for-you uses GET HomeTimeline, following uses POST HomeLatestTimeline @@ -137,7 +137,7 @@ cli({ site: 'twitter', name: 'timeline', access: 'read', - description: 'Fetch Twitter timeline (for-you or following)', + description: 'Fetch the logged-in user\'s home timeline (for-you algorithmic feed by default; pass --type following for the chronological feed of accounts you follow)', domain: 'x.com', strategy: Strategy.COOKIE, browser: true, @@ -146,9 +146,10 @@ cli({ name: 'type', default: 'for-you', choices: ['for-you', 'following'], - help: 'Timeline type: for-you (algorithmic) or following (chronological)', + help: 'Which home-timeline feed to read. Default for-you (algorithmic). Use following for the chronological feed of accounts you follow.', }, - { name: 'limit', type: 'int', default: 20 }, + { name: 'limit', type: 'int', default: 20, help: 'Maximum number of tweets to return (default 20).' }, + { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the timeline by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps X\'s native ordering.' }, ], columns: ['id', 'author', 'text', 'likes', 'retweets', 'replies', 'views', 'created_at', 'url', 'has_media', 'media_urls'], func: async (page, kwargs) => { @@ -168,7 +169,7 @@ cli({ const queryId = await resolveTwitterQueryId(page, endpoint, fallbackQueryId); // Build auth headers const headers = JSON.stringify({ - Authorization: `Bearer ${decodeURIComponent(BEARER_TOKEN)}`, + Authorization: `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, 'X-Csrf-Token': ct0, 'X-Twitter-Auth-Type': 'OAuth2Session', 'X-Twitter-Active-User': 'yes', @@ -196,7 +197,8 @@ cli({ break; cursor = nextCursor; } - return allTweets.slice(0, limit); + const trimmed = allTweets.slice(0, limit); + return applyTopByEngagement(trimmed, kwargs['top-by-engagement']); }, }); export const __test__ = { diff --git a/clis/twitter/tweets.js b/clis/twitter/tweets.js index f464c0010..38092e185 100644 --- a/clis/twitter/tweets.js +++ b/clis/twitter/tweets.js @@ -1,8 +1,8 @@ import { cli, Strategy } from '@jackwener/opencli/registry'; import { AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors'; import { resolveTwitterQueryId, sanitizeQueryId, extractMedia } from './shared.js'; +import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js'; -const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; const USER_TWEETS_QUERY_ID = '6fWQaBPK51aGyC_VC7t9GQ'; const USER_BY_SCREEN_NAME_QUERY_ID = 'IGgvgiOx4QZndDHuD3x9TQ'; @@ -152,6 +152,7 @@ cli({ args: [ { name: 'username', type: 'string', positional: true, required: true, help: 'Twitter screen name (with or without @)' }, { name: 'limit', type: 'int', default: 20, help: 'Max tweets to return' }, + { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the tweets by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the chronological ordering.' }, ], columns: ['id', 'author', 'created_at', 'is_retweet', 'text', 'likes', 'retweets', 'replies', 'views', 'url', 'has_media', 'media_urls'], func: async (page, kwargs) => { @@ -171,7 +172,7 @@ cli({ const userByScreenNameQueryId = await resolveTwitterQueryId(page, 'UserByScreenName', USER_BY_SCREEN_NAME_QUERY_ID); const headers = JSON.stringify({ - 'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`, + 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, 'X-Csrf-Token': ct0, 'X-Twitter-Auth-Type': 'OAuth2Session', 'X-Twitter-Active-User': 'yes', @@ -207,7 +208,8 @@ cli({ } if (all.length === 0) throw new EmptyResultError(`@${username} has no recent tweets`, 'Account may be private or suspended'); - return all.slice(0, limit); + const trimmed = all.slice(0, limit); + return applyTopByEngagement(trimmed, kwargs['top-by-engagement']); }, }); diff --git a/clis/twitter/unbookmark.js b/clis/twitter/unbookmark.js index 5800160bb..1b5b58853 100644 --- a/clis/twitter/unbookmark.js +++ b/clis/twitter/unbookmark.js @@ -1,5 +1,7 @@ import { CommandExecutionError } from '@jackwener/opencli/errors'; import { cli, Strategy } from '@jackwener/opencli/registry'; +import { parseTweetUrl, buildTwitterArticleScopeSource } from './shared.js'; + cli({ site: 'twitter', name: 'unbookmark', @@ -15,21 +17,25 @@ cli({ func: async (page, kwargs) => { if (!page) throw new CommandExecutionError('Browser session required for twitter unbookmark'); - await page.goto(kwargs.url); + const target = parseTweetUrl(kwargs.url); + await page.goto(target.url); await page.wait({ selector: '[data-testid="primaryColumn"]' }); const result = await page.evaluate(`(async () => { try { + ${buildTwitterArticleScopeSource(target.id)} let attempts = 0; let removeBtn = null; + let targetArticle = null; while (attempts < 20) { - // Check if not bookmarked - const bookmarkBtn = document.querySelector('[data-testid="bookmark"]'); + targetArticle = findTargetArticle(); + // Check if not bookmarked (already removed) + const bookmarkBtn = targetArticle?.querySelector('[data-testid="bookmark"]'); if (bookmarkBtn) { return { ok: true, message: 'Tweet is not bookmarked (already removed).' }; } - removeBtn = document.querySelector('[data-testid="removeBookmark"]'); + removeBtn = targetArticle?.querySelector('[data-testid="removeBookmark"]') || null; if (removeBtn) break; await new Promise(r => setTimeout(r, 500)); @@ -37,14 +43,15 @@ cli({ } if (!removeBtn) { - return { ok: false, message: 'Could not find Remove Bookmark button. Are you logged in?' }; + return { ok: false, message: 'Could not find Remove Bookmark button on the requested tweet. Are you logged in?' }; } removeBtn.click(); await new Promise(r => setTimeout(r, 1000)); // Verify - const verify = document.querySelector('[data-testid="bookmark"]'); + const verifyArticle = findTargetArticle() || targetArticle; + const verify = verifyArticle?.querySelector('[data-testid="bookmark"]'); if (verify) { return { ok: true, message: 'Tweet successfully removed from bookmarks.' }; } else { diff --git a/clis/twitter/unbookmark.test.js b/clis/twitter/unbookmark.test.js new file mode 100644 index 000000000..1c50827bf --- /dev/null +++ b/clis/twitter/unbookmark.test.js @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest'; +import { ArgumentError, CommandExecutionError } from '@jackwener/opencli/errors'; +import { getRegistry } from '@jackwener/opencli/registry'; +import './unbookmark.js'; +import { createPageMock } from '../test-utils.js'; + +describe('twitter unbookmark command', () => { + it('navigates to the tweet URL and reports success when the unbookmark script confirms', async () => { + const cmd = getRegistry().get('twitter/unbookmark'); + expect(cmd?.func).toBeTypeOf('function'); + const page = createPageMock([ + { ok: true, message: 'Tweet successfully removed from bookmarks.' }, + ]); + const result = await cmd.func(page, { + url: 'https://x.com/alice/status/2040254679301718161', + }); + expect(page.goto).toHaveBeenCalledWith('https://x.com/alice/status/2040254679301718161'); + expect(page.wait).toHaveBeenNthCalledWith(1, { selector: '[data-testid="primaryColumn"]' }); + expect(page.wait).toHaveBeenNthCalledWith(2, 2); + const script = page.evaluate.mock.calls[0][0]; + // Idempotency probe: when already not bookmarked ([data-testid="bookmark"] present), + // the script returns ok:true with an "already removed" message. + expect(script).toContain("targetArticle?.querySelector('[data-testid=\"bookmark\"]')"); + expect(script).toContain("targetArticle?.querySelector('[data-testid=\"removeBookmark\"]')"); + expect(script).toContain('removeBtn.click()'); + // Article scoping comes from the shared helper (buildTwitterArticleScopeSource): + // emits __twHasLinkToTarget + __twGetStatusIdFromHref + the anchored + // tweet-path regex. JSDOM-level coverage lives in shared.test.js. + expect(script).toContain('__twHasLinkToTarget'); + expect(script).toContain('__twGetStatusIdFromHref'); + expect(script).toContain("document.querySelectorAll('article')"); + expect(result).toEqual([ + { status: 'success', message: 'Tweet successfully removed from bookmarks.' }, + ]); + }); + + it('returns a failed row without re-waiting when the unbookmark script reports a UI mismatch', async () => { + const cmd = getRegistry().get('twitter/unbookmark'); + const page = createPageMock([ + { + ok: false, + message: 'Could not find Remove Bookmark button on the requested tweet. Are you logged in?', + }, + ]); + const result = await cmd.func(page, { + url: 'https://x.com/alice/status/2040254679301718161', + }); + expect(result).toEqual([ + { + status: 'failed', + message: 'Could not find Remove Bookmark button on the requested tweet. Are you logged in?', + }, + ]); + expect(page.wait).toHaveBeenCalledTimes(1); + }); + + it('throws CommandExecutionError when no page is provided', async () => { + const cmd = getRegistry().get('twitter/unbookmark'); + await expect(cmd.func(undefined, { + url: 'https://x.com/alice/status/2040254679301718161', + })).rejects.toThrow(CommandExecutionError); + }); + + it('rejects invalid tweet URLs before navigation', async () => { + const cmd = getRegistry().get('twitter/unbookmark'); + const page = createPageMock([]); + await expect(cmd.func(page, { + url: 'http://x.com/alice/status/2040254679301718161', + })).rejects.toThrow(ArgumentError); + expect(page.goto).not.toHaveBeenCalled(); + expect(page.evaluate).not.toHaveBeenCalled(); + }); +}); diff --git a/clis/twitter/unlike.js b/clis/twitter/unlike.js index dbde29703..d0f0516d5 100644 --- a/clis/twitter/unlike.js +++ b/clis/twitter/unlike.js @@ -1,6 +1,6 @@ import { CommandExecutionError } from '@jackwener/opencli/errors'; import { cli, Strategy } from '@jackwener/opencli/registry'; -import { parseTweetUrl } from './shared.js'; +import { parseTweetUrl, buildTwitterArticleScopeSource } from './shared.js'; cli({ site: 'twitter', @@ -22,18 +22,11 @@ cli({ await page.wait({ selector: '[data-testid="primaryColumn"]' }); const result = await page.evaluate(`(async () => { try { - const tweetId = ${JSON.stringify(target.id)}; - const findTargetArticle = () => Array.from(document.querySelectorAll('article')).find((article) => - Array.from(article.querySelectorAll('a[href*="/status/"]')).some((link) => { - try { - const match = new URL(link.href, window.location.origin).pathname.match(/^\/(?:[^/]+|i)\/status\/(\d+)\/?$/); - return match?.[1] === tweetId; - } catch { - return false; - } - }) - ); - // Poll for the tweet to render + ${buildTwitterArticleScopeSource(target.id)} + // Poll for the tweet to render. State probes scoped to the article + // matching the requested status id — bare querySelector on a + // conversation page would silently grab the first article (e.g. + // the parent tweet) and unlike the wrong one. let attempts = 0; let likeBtn = null; let unlikeBtn = null; diff --git a/clis/twitter/unlike.test.js b/clis/twitter/unlike.test.js index 0c1bdedf7..b421eba2e 100644 --- a/clis/twitter/unlike.test.js +++ b/clis/twitter/unlike.test.js @@ -23,9 +23,12 @@ describe('twitter unlike command', () => { expect(script).toContain("targetArticle?.querySelector('[data-testid=\"like\"]')"); expect(script).toContain("targetArticle?.querySelector('[data-testid=\"unlike\"]')"); expect(script).toContain('unlikeBtn.click()'); + // Article scoping comes from the shared helper (buildTwitterArticleScopeSource): + // emits __twHasLinkToTarget + __twGetStatusIdFromHref + the anchored + // tweet-path regex. JSDOM-level coverage lives in shared.test.js. + expect(script).toContain('__twHasLinkToTarget'); + expect(script).toContain('__twGetStatusIdFromHref'); expect(script).toContain("document.querySelectorAll('article')"); - expect(script).toContain('match?.[1] === tweetId'); - expect(script).toContain("targetArticle?.querySelector('[data-testid=\"unlike\"]')"); expect(result).toEqual([ { status: 'success', message: 'Tweet successfully unliked.' }, ]); diff --git a/clis/twitter/unretweet.js b/clis/twitter/unretweet.js index b6b28e969..2aed853c3 100644 --- a/clis/twitter/unretweet.js +++ b/clis/twitter/unretweet.js @@ -1,6 +1,6 @@ import { CommandExecutionError } from '@jackwener/opencli/errors'; import { cli, Strategy } from '@jackwener/opencli/registry'; -import { parseTweetUrl } from './shared.js'; +import { parseTweetUrl, buildTwitterArticleScopeSource } from './shared.js'; cli({ site: 'twitter', @@ -22,18 +22,11 @@ cli({ await page.wait({ selector: '[data-testid="primaryColumn"]' }); const result = await page.evaluate(`(async () => { try { - const tweetId = ${JSON.stringify(target.id)}; - const findTargetArticle = () => Array.from(document.querySelectorAll('article')).find((article) => - Array.from(article.querySelectorAll('a[href*="/status/"]')).some((link) => { - try { - const match = new URL(link.href, window.location.origin).pathname.match(/^\/(?:[^/]+|i)\/status\/(\d+)\/?$/); - return match?.[1] === tweetId; - } catch { - return false; - } - }) - ); - // Poll for the tweet to render + ${buildTwitterArticleScopeSource(target.id)} + // Poll for the tweet to render. State probes scoped to the article + // matching the requested status id — bare querySelector on a + // conversation page would silently grab the first article (e.g. + // the parent tweet) and unretweet the wrong one. let attempts = 0; let retweetBtn = null; let unretweetBtn = null; @@ -62,7 +55,9 @@ cli({ // Step 1: click Unretweet button → opens menu unretweetBtn.click(); - // Step 2: wait for the confirm menu item to appear, then click it + // Step 2: wait for and click the confirm menu item. The confirm + // popover renders at the document root, not inside the article, + // so this lookup is intentionally document-scoped. let confirmBtn = null; for (let i = 0; i < 20; i++) { await new Promise(r => setTimeout(r, 250)); diff --git a/clis/twitter/unretweet.test.js b/clis/twitter/unretweet.test.js index 97d5a395b..f94fe8880 100644 --- a/clis/twitter/unretweet.test.js +++ b/clis/twitter/unretweet.test.js @@ -24,8 +24,12 @@ describe('twitter unretweet command', () => { expect(script).toContain('unretweetBtn.click()'); expect(script).toContain("document.querySelector('[data-testid=\"unretweetConfirm\"]')"); expect(script).toContain('confirmBtn.click()'); + // Article scoping comes from the shared helper (buildTwitterArticleScopeSource): + // emits __twHasLinkToTarget + __twGetStatusIdFromHref + the anchored + // tweet-path regex. JSDOM-level coverage lives in shared.test.js. + expect(script).toContain('__twHasLinkToTarget'); + expect(script).toContain('__twGetStatusIdFromHref'); expect(script).toContain("document.querySelectorAll('article')"); - expect(script).toContain('match?.[1] === tweetId'); expect(script).toContain("targetArticle?.querySelector('[data-testid=\"unretweet\"]')"); // Idempotency probe: when already not retweeted ([data-testid="retweet"] present), // the script returns ok:true with an "already removed" message. diff --git a/clis/twitter/utils.js b/clis/twitter/utils.js new file mode 100644 index 000000000..2db93db81 --- /dev/null +++ b/clis/twitter/utils.js @@ -0,0 +1,286 @@ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { ArgumentError } from '@jackwener/opencli/errors'; + +/** + * Public read-only Twitter web bearer token used by the GraphQL endpoints we + * call from the page context. This is the same token the Twitter web app + * itself uses; centralising it here keeps the 12+ GraphQL adapters from + * drifting when X rotates the value. + */ +export const TWITTER_BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; + +/** File-input selector used by the X /compose/post route for both posts and replies. */ +export const COMPOSER_FILE_INPUT_SELECTOR = 'input[type="file"][data-testid="fileInput"]'; + +/** Image formats the X composer accepts. */ +export const SUPPORTED_IMAGE_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp']); + +/** 20 MB hard cap. Twitter allows ~5MB images / 15MB GIFs; 20MB is a safety net. */ +export const MAX_IMAGE_SIZE_BYTES = 20 * 1024 * 1024; + +const CONTENT_TYPE_TO_EXTENSION = { + 'image/jpeg': '.jpg', + 'image/jpg': '.jpg', + 'image/png': '.png', + 'image/gif': '.gif', + 'image/webp': '.webp', +}; + +/** + * Validate a single image path. Throws {@link ArgumentError} on bad input + * (typed input failure surfaces before any browser interaction). + * + * @param {string} imagePath - Local filesystem path, may be relative. + * @returns {string} Absolute resolved path. + */ +export function resolveImagePath(imagePath) { + const absPath = path.resolve(imagePath); + if (!fs.existsSync(absPath)) { + throw new ArgumentError(`Image file not found: ${absPath}`); + } + const ext = path.extname(absPath).toLowerCase(); + if (!SUPPORTED_IMAGE_EXTENSIONS.has(ext)) { + throw new ArgumentError(`Unsupported image format "${ext}". Supported: jpg, jpeg, png, gif, webp`); + } + const stat = fs.statSync(absPath); + if (stat.size > MAX_IMAGE_SIZE_BYTES) { + throw new ArgumentError(`Image too large: ${(stat.size / 1024 / 1024).toFixed(1)} MB (max ${MAX_IMAGE_SIZE_BYTES / 1024 / 1024} MB)`); + } + return absPath; +} + +/** + * Resolve the file extension to use when persisting a remote image: prefer + * Content-Type, fall back to URL pathname. + */ +export function resolveImageExtension(url, contentType) { + const normalizedContentType = (contentType || '').split(';')[0].trim().toLowerCase(); + if (normalizedContentType && CONTENT_TYPE_TO_EXTENSION[normalizedContentType]) { + return CONTENT_TYPE_TO_EXTENSION[normalizedContentType]; + } + try { + const pathname = new URL(url).pathname; + const ext = path.extname(pathname).toLowerCase(); + if (SUPPORTED_IMAGE_EXTENSIONS.has(ext)) + return ext; + } catch { + // Fall through to the final error below. + } + throw new ArgumentError( + `Unsupported remote image format "${normalizedContentType || 'unknown'}". Supported: jpg, jpeg, png, gif, webp`, + ); +} + +/** + * Download a remote image to a per-call tmp directory. Returns the absolute + * path on success. Caller owns the tmp dir and must clean it up. Throws + * {@link ArgumentError} on bad input or download failure. + * + * @returns {Promise<{ absPath: string, cleanupDir: string }>} + */ +export async function downloadRemoteImage(imageUrl) { + let parsed; + try { + parsed = new URL(imageUrl); + } catch { + throw new ArgumentError(`Invalid image URL: ${imageUrl}`); + } + if (!/^https?:$/.test(parsed.protocol)) { + throw new ArgumentError(`Unsupported image URL protocol: ${parsed.protocol}`); + } + const response = await fetch(imageUrl); + if (!response.ok) { + throw new ArgumentError(`Image download failed: HTTP ${response.status}`); + } + const contentLength = Number(response.headers.get('content-length') || '0'); + if (contentLength > MAX_IMAGE_SIZE_BYTES) { + throw new ArgumentError(`Image too large: ${(contentLength / 1024 / 1024).toFixed(1)} MB (max ${MAX_IMAGE_SIZE_BYTES / 1024 / 1024} MB)`); + } + const ext = resolveImageExtension(imageUrl, response.headers.get('content-type')); + const cleanupDir = fs.mkdtempSync(path.join(os.tmpdir(), 'opencli-twitter-')); + const absPath = path.join(cleanupDir, `image${ext}`); + const buffer = Buffer.from(await response.arrayBuffer()); + if (buffer.byteLength > MAX_IMAGE_SIZE_BYTES) { + fs.rmSync(cleanupDir, { recursive: true, force: true }); + throw new ArgumentError(`Image too large: ${(buffer.byteLength / 1024 / 1024).toFixed(1)} MB (max ${MAX_IMAGE_SIZE_BYTES / 1024 / 1024} MB)`); + } + fs.writeFileSync(absPath, buffer); + return { absPath, cleanupDir }; +} + +/** + * Attach a single image to the current /compose/post composer. Tries the + * native CDP file-input bridge first; falls back to a base64 DataTransfer + * shim if the bridge is missing or rejects with "Unknown action" / + * "not supported". Throws on hard failures. + * + * After upload it polls the DOM briefly to confirm the preview thumbnail + * actually rendered — without this, a 200 from setFileInput could mask a + * silent-no-attachment post. + * + * @param {object} page - OpenCLI page handle. + * @param {string} absImagePath - Already-validated absolute path. + * @param {string} [fileInputSelector] - Override (post.js historically used + * the same selector; default matches the X composer route). + */ +export async function attachComposerImage(page, absImagePath, fileInputSelector = COMPOSER_FILE_INPUT_SELECTOR) { + let uploaded = false; + if (page.setFileInput) { + try { + await page.setFileInput([absImagePath], fileInputSelector); + uploaded = true; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (!msg.includes('Unknown action') && !msg.includes('not supported')) { + throw new Error(`Image upload failed: ${msg}`); + } + // setFileInput not supported by extension — fall through to base64 fallback. + } + } + if (!uploaded) { + const ext = path.extname(absImagePath).toLowerCase(); + const mimeType = ext === '.png' + ? 'image/png' + : ext === '.gif' + ? 'image/gif' + : ext === '.webp' + ? 'image/webp' + : 'image/jpeg'; + const base64 = fs.readFileSync(absImagePath).toString('base64'); + if (base64.length > 500_000) { + console.warn(`[warn] Image base64 payload is ${(base64.length / 1024 / 1024).toFixed(1)}MB. ` + + 'This may fail with the browser bridge. Update the extension to v1.6+ for CDP-based upload, ' + + 'or compress the image before attaching.'); + } + const upload = await page.evaluate(` + (() => { + const input = document.querySelector(${JSON.stringify(fileInputSelector)}); + if (!input) return { ok: false, error: 'No file input found on page' }; + + const binary = atob(${JSON.stringify(base64)}); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); + + const dt = new DataTransfer(); + const blob = new Blob([bytes], { type: ${JSON.stringify(mimeType)} }); + dt.items.add(new File([blob], ${JSON.stringify(path.basename(absImagePath))}, { type: ${JSON.stringify(mimeType)} })); + + Object.defineProperty(input, 'files', { value: dt.files, writable: false }); + input.dispatchEvent(new Event('change', { bubbles: true })); + input.dispatchEvent(new Event('input', { bubbles: true })); + return { ok: true }; + })() + `); + if (!upload?.ok) { + throw new Error(`Image upload failed: ${upload?.error ?? 'unknown error'}`); + } + } + await page.wait(2); + const uploadState = await page.evaluate(` + (() => { + const previewCount = document.querySelectorAll( + '[data-testid="attachments"] img, [data-testid="attachments"] video, [data-testid="tweetPhoto"]' + ).length; + const hasMedia = previewCount > 0 + || !!document.querySelector('[data-testid="attachments"]') + || !!Array.from(document.querySelectorAll('button,[role="button"]')).find((el) => + /remove media|remove image|remove/i.test((el.getAttribute('aria-label') || '') + ' ' + (el.textContent || '')) + ); + return { ok: hasMedia, previewCount }; + })() + `); + if (!uploadState?.ok) { + throw new Error('Image upload failed: preview did not appear.'); + } +} + +// ── Engagement scoring (P3) ──────────────────────────────────────────── +// +// Used by tweet-shaped read commands (search / timeline / likes / bookmarks / +// list-tweets / tweets / thread). Lets callers ask for the top-N tweets by +// weighted engagement instead of chronological order, so an agent skimming a +// noisy timeline can surface the actually-interesting tweets first. +// +// The weights bias toward "active engagement": bookmarks > retweets > replies +// > likes > views. Views are log-dampened because they often dwarf all other +// signals by 2–4 orders of magnitude on viral tweets and would otherwise +// drown out the active signals. +// +// Pure synchronous — exported via __test__ for unit coverage. Missing fields +// (some adapters don't surface views/replies/bookmarks) coerce to 0 so the +// formula stays well-defined across every read command's row shape. + +const ENGAGEMENT_WEIGHTS = Object.freeze({ + likes: 1, + retweets: 3, + replies: 2, + bookmarks: 5, + viewsLog: 0.5, +}); + +/** + * Compute the weighted engagement score for a tweet-shaped row. + * + * Formula: likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5 + * + * - String fields (e.g. views: '12345') are coerced via Number(); non-numeric + * strings become 0 instead of NaN-poisoning the score. + * - log10(views+1) so views=0 maps to 0 (not -Infinity). + * - Missing fields default to 0 — search returns no `replies`/`bookmarks`, + * bookmarks returns no `views`/`replies`, etc. + * + * @param {Record} row + * @returns {number} Score, rounded to 2 decimals for stable test fixtures. + */ +export function computeEngagementScore(row) { + if (!row || typeof row !== 'object') return 0; + const num = (key) => { + const raw = row[key]; + if (raw === undefined || raw === null) return 0; + const n = Number(raw); + return Number.isFinite(n) ? Math.max(0, n) : 0; + }; + const score + = num('likes') * ENGAGEMENT_WEIGHTS.likes + + num('retweets') * ENGAGEMENT_WEIGHTS.retweets + + num('replies') * ENGAGEMENT_WEIGHTS.replies + + num('bookmarks') * ENGAGEMENT_WEIGHTS.bookmarks + + Math.log10(num('views') + 1) * ENGAGEMENT_WEIGHTS.viewsLog; + return Math.round(score * 100) / 100; +} + +/** + * Apply --top-by-engagement post-processing. When `topN > 0` the rows are + * sorted DESCENDING by computeEngagementScore() and trimmed to the top N. + * When `topN <= 0` (the default), rows are returned unchanged so adapters + * that don't pass the flag stay backward compatible. + * + * Stable for ties: rows with the same score retain their original order + * (Array.prototype.sort is guaranteed stable in V8 since 2018). + * + * @param {Array>} rows + * @param {number} topN + * @returns {Array>} + */ +export function applyTopByEngagement(rows, topN) { + if (!Array.isArray(rows) || rows.length === 0) return rows; + const n = Number(topN); + if (!Number.isFinite(n) || n <= 0) return rows; + return rows + .map((row, idx) => ({ row, idx, score: computeEngagementScore(row) })) + .sort((a, b) => b.score - a.score || a.idx - b.idx) + .slice(0, Math.floor(n)) + .map(entry => entry.row); +} + +export const __test__ = { + resolveImagePath, + resolveImageExtension, + downloadRemoteImage, + attachComposerImage, + computeEngagementScore, + applyTopByEngagement, + ENGAGEMENT_WEIGHTS, +}; diff --git a/clis/twitter/utils.test.js b/clis/twitter/utils.test.js new file mode 100644 index 000000000..ae6cf5729 --- /dev/null +++ b/clis/twitter/utils.test.js @@ -0,0 +1,169 @@ +import { describe, expect, it } from 'vitest'; +import { __test__ } from './utils.js'; + +const { computeEngagementScore, applyTopByEngagement, ENGAGEMENT_WEIGHTS } = __test__; + +describe('computeEngagementScore', () => { + it('returns 0 for empty / nullish rows', () => { + expect(computeEngagementScore(null)).toBe(0); + expect(computeEngagementScore(undefined)).toBe(0); + expect(computeEngagementScore({})).toBe(0); + }); + + it('weights likes ×1', () => { + expect(computeEngagementScore({ likes: 10 })).toBe(10); + }); + + it('weights retweets ×3', () => { + expect(computeEngagementScore({ retweets: 5 })).toBe(15); + }); + + it('weights replies ×2', () => { + expect(computeEngagementScore({ replies: 4 })).toBe(8); + }); + + it('weights bookmarks ×5', () => { + expect(computeEngagementScore({ bookmarks: 6 })).toBe(30); + }); + + it('log-dampens views (log10(v+1) × 0.5)', () => { + // log10(99+1) * 0.5 = 1.0 + expect(computeEngagementScore({ views: 99 })).toBeCloseTo(1.0, 2); + // log10(0+1) * 0.5 = 0 + expect(computeEngagementScore({ views: 0 })).toBe(0); + }); + + it('coerces string-typed views (search/timeline returns views as a string)', () => { + // log10(9999+1) * 0.5 = 2.0 + expect(computeEngagementScore({ views: '9999' })).toBeCloseTo(2.0, 2); + }); + + it('treats non-numeric strings as 0 instead of NaN-poisoning the score', () => { + expect(computeEngagementScore({ likes: 'abc', retweets: 5 })).toBe(15); + }); + + it('clamps negative values at 0 (defensive against bogus payloads)', () => { + expect(computeEngagementScore({ likes: -100, retweets: 2 })).toBe(6); + }); + + it('combines all signals additively', () => { + // likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5 + // 10 + 30 + 8 + 25 + log10(1000)*0.5 = 73 + 1.5 = 74.5 + const row = { likes: 10, retweets: 10, replies: 4, bookmarks: 5, views: 999 }; + expect(computeEngagementScore(row)).toBeCloseTo(74.5, 2); + }); + + it('rounds to 2 decimal places for stable test fixtures', () => { + // log10(1+1) * 0.5 = 0.5 * log10(2) ≈ 0.150515 + const score = computeEngagementScore({ views: 1 }); + expect(score).toBe(0.15); + }); + + it('handles real search-row shape (no replies/bookmarks columns)', () => { + const searchRow = { + id: '123', + author: 'alice', + text: 'hi', + likes: 100, + views: '9999', + }; + // 100 + log10(10000)*0.5 = 100 + 2.0 = 102.0 + expect(computeEngagementScore(searchRow)).toBeCloseTo(102.0, 2); + }); + + it('handles real bookmarks-row shape (no views/replies columns)', () => { + const bookmarkRow = { + id: '123', + author: 'alice', + text: 'hi', + likes: 50, + retweets: 10, + bookmarks: 3, + }; + // 50 + 30 + 15 = 95 + expect(computeEngagementScore(bookmarkRow)).toBe(95); + }); + + it('exposes the documented weight table', () => { + expect(ENGAGEMENT_WEIGHTS).toEqual({ + likes: 1, + retweets: 3, + replies: 2, + bookmarks: 5, + viewsLog: 0.5, + }); + }); +}); + +describe('applyTopByEngagement', () => { + const rows = [ + { id: 'a', likes: 10 }, + { id: 'b', likes: 50 }, + { id: 'c', likes: 30 }, + { id: 'd', likes: 100 }, + { id: 'e', likes: 5 }, + ]; + + it('returns rows unchanged when topN is 0 (default)', () => { + expect(applyTopByEngagement(rows, 0)).toBe(rows); + }); + + it('returns rows unchanged when topN is negative', () => { + expect(applyTopByEngagement(rows, -3)).toBe(rows); + }); + + it('returns rows unchanged when topN is non-numeric', () => { + expect(applyTopByEngagement(rows, 'foo')).toBe(rows); + expect(applyTopByEngagement(rows, null)).toBe(rows); + expect(applyTopByEngagement(rows, undefined)).toBe(rows); + }); + + it('sorts descending by score and trims to top N when topN > 0', () => { + const result = applyTopByEngagement(rows, 3); + expect(result.map(r => r.id)).toEqual(['d', 'b', 'c']); + }); + + it('returns all rows when topN exceeds row count', () => { + const result = applyTopByEngagement(rows, 99); + expect(result.map(r => r.id)).toEqual(['d', 'b', 'c', 'a', 'e']); + }); + + it('floors fractional topN', () => { + const result = applyTopByEngagement(rows, 2.9); + expect(result.map(r => r.id)).toEqual(['d', 'b']); + }); + + it('is stable for ties (preserves original order)', () => { + const tieRows = [ + { id: 'first', likes: 10 }, + { id: 'second', likes: 10 }, + { id: 'third', likes: 10 }, + { id: 'fourth', likes: 100 }, + ]; + const result = applyTopByEngagement(tieRows, 4); + // 'fourth' first, then ties retain original order + expect(result.map(r => r.id)).toEqual(['fourth', 'first', 'second', 'third']); + }); + + it('handles empty / non-array input gracefully', () => { + expect(applyTopByEngagement([], 5)).toEqual([]); + expect(applyTopByEngagement(null, 5)).toBeNull(); + expect(applyTopByEngagement(undefined, 5)).toBeUndefined(); + }); + + it('does not mutate the input array', () => { + const before = [...rows]; + applyTopByEngagement(rows, 2); + expect(rows).toEqual(before); + }); + + it('mixes signals correctly when ranking', () => { + // bookmark-heavy row should beat like-heavy row even if likes are higher + const mixed = [ + { id: 'likes-only', likes: 100 }, // score = 100 + { id: 'bookmark-heavy', likes: 30, bookmarks: 20 }, // score = 30 + 100 = 130 + ]; + const result = applyTopByEngagement(mixed, 2); + expect(result.map(r => r.id)).toEqual(['bookmark-heavy', 'likes-only']); + }); +}); diff --git a/scripts/typed-error-lint-baseline.json b/scripts/typed-error-lint-baseline.json index a69a5f82d..38f9a51d6 100644 --- a/scripts/typed-error-lint-baseline.json +++ b/scripts/typed-error-lint-baseline.json @@ -615,11 +615,19 @@ "text": "rn_need: String(Math.min(Math.max(limit + 10, 10), 30)),", "occurrence": 0 }, + { + "rule": "silent-clamp", + "command": "twitter/bookmark-folder", + "file": "clis/twitter/bookmark-folder.js", + "line": 161, + "text": "const fetchCount = Math.min(100, limit - allTweets.length + 10);", + "occurrence": 0 + }, { "rule": "silent-clamp", "command": "twitter/bookmarks", "file": "clis/twitter/bookmarks.js", - "line": 155, + "line": 156, "text": "const fetchCount = Math.min(100, limit - allTweets.length + 10);", "occurrence": 0 }, @@ -627,7 +635,7 @@ "rule": "silent-clamp", "command": "twitter/following", "file": "clis/twitter/following.js", - "line": 209, + "line": 215, "text": "const fetchCount = Math.min(50, limit - allUsers.length + 10);", "occurrence": 0 }, @@ -635,7 +643,7 @@ "rule": "silent-clamp", "command": "twitter/likes", "file": "clis/twitter/likes.js", - "line": 194, + "line": 195, "text": "const fetchCount = Math.min(100, limit - allTweets.length + 10);", "occurrence": 0 }, @@ -643,7 +651,7 @@ "rule": "silent-clamp", "command": "twitter/list-tweets", "file": "clis/twitter/list-tweets.js", - "line": 167, + "line": 168, "text": "const fetchCount = Math.min(100, limit - allTweets.length + 10);", "occurrence": 0 }, @@ -651,7 +659,7 @@ "rule": "silent-clamp", "command": "twitter/timeline", "file": "clis/twitter/timeline.js", - "line": 181, + "line": 182, "text": "const fetchCount = Math.min(40, limit - allTweets.length + 5); // over-fetch slightly for promoted filtering", "occurrence": 0 }, @@ -659,7 +667,7 @@ "rule": "silent-clamp", "command": "twitter/tweets", "file": "clis/twitter/tweets.js", - "line": 193, + "line": 194, "text": "const fetchCount = Math.min(100, limit - all.length + 10);", "occurrence": 0 }, @@ -667,7 +675,7 @@ "rule": "silent-clamp", "command": "twitter/tweets", "file": "clis/twitter/tweets.js", - "line": 158, + "line": 159, "text": "const limit = Math.max(1, Math.min(200, kwargs.limit || 20));", "occurrence": 0 }, @@ -1203,7 +1211,7 @@ "rule": "silent-sentinel", "command": "twitter/article", "file": "clis/twitter/article.js", - "line": 104, + "line": 105, "text": "const screenName = user?.legacy?.screen_name || user?.core?.screen_name || 'unknown';", "occurrence": 0 }, @@ -1279,19 +1287,11 @@ "text": "const user = lines[0] || 'Unknown';", "occurrence": 0 }, - { - "rule": "silent-sentinel", - "command": "twitter/reply", - "file": "clis/twitter/reply.js", - "line": 68, - "text": "throw new Error(`Unsupported remote image format \"${normalizedContentType || 'unknown'}\". ` +", - "occurrence": 0 - }, { "rule": "silent-sentinel", "command": "twitter/search", "file": "clis/twitter/search.js", - "line": 156, + "line": 296, "text": "author: tweetUser?.core?.screen_name || tweetUser?.legacy?.screen_name || 'unknown',", "occurrence": 0 }, @@ -1411,7 +1411,7 @@ "rule": "silent-sentinel", "command": "xiaohongshu/publish", "file": "clis/xiaohongshu/publish.js", - "line": 516, + "line": 529, "text": "throw new Error(`Image injection failed: ${upload.error ?? 'unknown'}. ` +", "occurrence": 0 },