From a91bd4fc85afc98f84fd0a76b3631e0b571d33a3 Mon Sep 17 00:00:00 2001 From: aldosch Date: Sun, 14 Sep 2025 21:39:25 +1000 Subject: [PATCH 1/4] fix issue #1 by removing redundant await --- src/login.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/login.ts b/src/login.ts index 3796950..ac76bc4 100644 --- a/src/login.ts +++ b/src/login.ts @@ -4,8 +4,6 @@ export async function login(page: Page, email: string): Promise { console.log("Navigating to Perplexity..."); await page.goto("https://www.perplexity.ai/"); - await page.click("button::-p-text('Accept All Cookies')"); - // Wait for email input and enter credentials await page.waitForSelector('input[type="email"]'); await page.type('input[type="email"]', email); From b72694d91fdf1077b1f53768bce6134e19d174e4 Mon Sep 17 00:00:00 2001 From: aldosch Date: Sun, 14 Sep 2025 22:16:05 +1000 Subject: [PATCH 2/4] create done.json if none exists --- src/utils.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index b1c5bdd..a3f24d2 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -5,7 +5,16 @@ export async function loadDoneFile(doneFilePath: string): Promise { try { const content = await fs.readFile(doneFilePath, "utf-8"); return JSON.parse(content); - } catch (error) { + } catch (error: any) { + if (error.code === "ENOENT") { + const defaultDoneFile = { processedUrls: [] }; + console.log(`Created empty ${doneFilePath}`); + await fs.writeFile( + doneFilePath, + JSON.stringify(defaultDoneFile, null, 2), + ); + return defaultDoneFile; + } console.error(`Error loading done file ${doneFilePath}:`, error); return { processedUrls: [] }; } @@ -13,7 +22,7 @@ export async function loadDoneFile(doneFilePath: string): Promise { export async function saveDoneFile( doneFile: DoneFile, - doneFilePath: string + doneFilePath: string, ): Promise { await fs.writeFile(doneFilePath, JSON.stringify(doneFile, null, 2)); } From 47f975e764b904be29de1c438a2ce30429d9882a Mon Sep 17 00:00:00 2001 From: aldosch Date: Sun, 14 Sep 2025 23:19:24 +1000 Subject: [PATCH 3/4] add --sort-order, fix thread selector, add thread scroll logging --- src/cli.ts | 19 +++++++++ src/exportLibrary.ts | 3 +- src/listConversations.ts | 92 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 106 insertions(+), 8 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 4a61d03..e3ab8dc 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -15,16 +15,35 @@ program "Done file location (tracks which URLs have been downloaded before)", "done.json" ) + .option( + "-s, --sort-order ", + "Sort order: 'newest' or 'oldest'", + "newest" + ) .requiredOption("-e, --email ", "Perplexity email") .parse(); const options = program.opts(); async function main(): Promise { + // Normalize and validate sort order + let sortOrder = options.sortOrder; + if (sortOrder === "o" || sortOrder === "old") { + sortOrder = "oldest"; + } else if (sortOrder === "n" || sortOrder === "new") { + sortOrder = "newest"; + } + + if (sortOrder !== "newest" && sortOrder !== "oldest") { + console.error("Error: Sort order must be 'newest' or 'oldest'"); + process.exit(1); + } + await exportLibrary({ outputDir: options.output, doneFilePath: options.doneFile, email: options.email, + sortOrder: sortOrder, }); } diff --git a/src/exportLibrary.ts b/src/exportLibrary.ts index b9ada3f..83a40cf 100644 --- a/src/exportLibrary.ts +++ b/src/exportLibrary.ts @@ -12,6 +12,7 @@ export interface ExportLibraryOptions { outputDir: string; doneFilePath: string; email: string; + sortOrder?: 'newest' | 'oldest'; } export default async function exportLibrary(options: ExportLibraryOptions) { @@ -35,7 +36,7 @@ export default async function exportLibrary(options: ExportLibraryOptions) { const page = await browser.newPage(); await login(page, options.email); - const conversations = await getConversations(page, doneFile); + const conversations = await getConversations(page, doneFile, options.sortOrder || 'newest'); console.log(`Found ${conversations.length} new conversations to process`); diff --git a/src/listConversations.ts b/src/listConversations.ts index a995d78..b7cd444 100644 --- a/src/listConversations.ts +++ b/src/listConversations.ts @@ -2,6 +2,77 @@ import { Page } from "puppeteer"; import { Conversation, DoneFile } from "./types"; import { sleep } from "./utils"; +async function setSortOrder(page: Page, sortOrder: 'newest' | 'oldest'): Promise { + console.log(`Setting sort order to: ${sortOrder}`); + + try { + // Find the sort button by looking for text containing either "Sort: Newest" or "Sort: Oldest" + await page.waitForFunction( + () => { + const buttons = Array.from(document.querySelectorAll('button')); + return buttons.some(button => { + const text = button.textContent || ''; + return text.includes('Sort: Newest') || text.includes('Sort: Oldest'); + }); + }, + { timeout: 10000 } + ); + + // Click the sort button + await page.evaluate(() => { + const buttons = Array.from(document.querySelectorAll('button')); + const sortButton = buttons.find(button => { + const text = button.textContent || ''; + return text.includes('Sort: Newest') || text.includes('Sort: Oldest'); + }); + if (sortButton) { + (sortButton as HTMLElement).click(); + } else { + throw new Error('Could not find sort button'); + } + }); + + console.log("Clicked sort dropdown button"); + + // Wait for dropdown menu to appear + await sleep(1000); + + // Find and click the appropriate sort option + const targetText = sortOrder === 'newest' ? 'Newest First' : 'Oldest First'; + + // Wait for the dropdown menu and find the option by text + await page.waitForFunction( + (text) => { + const elements = Array.from(document.querySelectorAll('[role="menuitem"]')); + return elements.some(el => el.textContent?.includes(text)); + }, + { timeout: 5000 }, + targetText + ); + + // Click the option containing the target text + await page.evaluate((text) => { + const elements = Array.from(document.querySelectorAll('[role="menuitem"]')); + const target = elements.find(el => el.textContent?.includes(text)); + if (target) { + (target as HTMLElement).click(); + } else { + throw new Error(`Could not find menu item with text: ${text}`); + } + }, targetText); + + console.log(`Clicked on "${targetText}" option`); + + // Wait for UI to update + await sleep(2000); + + console.log("Sort order has been set successfully"); + } catch (error) { + console.error(`Failed to set sort order: ${error}`); + throw error; + } +} + export async function scrollToBottomOfConversations( page: Page, doneFile: DoneFile @@ -48,18 +119,23 @@ export async function scrollToBottomOfConversations( export async function getConversations( page: Page, - doneFile: DoneFile + doneFile: DoneFile, + sortOrder: 'newest' | 'oldest' = 'newest' ): Promise { console.log("Navigating to library..."); await page.goto("https://www.perplexity.ai/library"); - await page.waitForSelector('div[data-testid="thread-title"]'); + await page.waitForSelector('div[data-testid^="thread-title-"]'); + + // Set the sort order before scrolling + await setSortOrder(page, sortOrder); + await scrollToBottomOfConversations(page, doneFile); // Get all conversation links const conversations = await page.evaluate(() => { const items = Array.from( - document.querySelectorAll('div[data-testid="thread-title"]') + document.querySelectorAll('div[data-testid^="thread-title-"]') ).map((div: Element) => div.closest("a") as HTMLAnchorElement); return items.map((item) => ({ title: item.textContent?.trim() || "Untitled", @@ -67,8 +143,10 @@ export async function getConversations( })); }); - // Filter out already processed conversations and reverse the order - return conversations - .filter((conv) => !doneFile.processedUrls.includes(conv.url)) - .reverse(); + // Filter out already processed conversations + const filtered = conversations.filter((conv) => !doneFile.processedUrls.includes(conv.url)); + + // For "newest" sort order, reverse to process oldest first (as original behavior) + // For "oldest" sort order, keep the order as-is since we want to process oldest first + return sortOrder === 'newest' ? filtered.reverse() : filtered; } From 1be4294cd95ee8b0bb41677d3736e3216d18d37d Mon Sep 17 00:00:00 2001 From: aldosch Date: Sun, 14 Sep 2025 23:28:27 +1000 Subject: [PATCH 4/4] formatting --- src/listConversations.ts | 59 ++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/src/listConversations.ts b/src/listConversations.ts index b7cd444..09cb1d8 100644 --- a/src/listConversations.ts +++ b/src/listConversations.ts @@ -2,58 +2,63 @@ import { Page } from "puppeteer"; import { Conversation, DoneFile } from "./types"; import { sleep } from "./utils"; -async function setSortOrder(page: Page, sortOrder: 'newest' | 'oldest'): Promise { +async function setSortOrder( + page: Page, + sortOrder: "newest" | "oldest", +): Promise { console.log(`Setting sort order to: ${sortOrder}`); try { // Find the sort button by looking for text containing either "Sort: Newest" or "Sort: Oldest" await page.waitForFunction( () => { - const buttons = Array.from(document.querySelectorAll('button')); - return buttons.some(button => { - const text = button.textContent || ''; - return text.includes('Sort: Newest') || text.includes('Sort: Oldest'); + const buttons = Array.from(document.querySelectorAll("button")); + return buttons.some((button) => { + const text = button.textContent || ""; + return text.includes("Sort: Newest") || text.includes("Sort: Oldest"); }); }, - { timeout: 10000 } + { timeout: 10000 }, ); // Click the sort button await page.evaluate(() => { - const buttons = Array.from(document.querySelectorAll('button')); - const sortButton = buttons.find(button => { - const text = button.textContent || ''; - return text.includes('Sort: Newest') || text.includes('Sort: Oldest'); + const buttons = Array.from(document.querySelectorAll("button")); + const sortButton = buttons.find((button) => { + const text = button.textContent || ""; + return text.includes("Sort: Newest") || text.includes("Sort: Oldest"); }); if (sortButton) { (sortButton as HTMLElement).click(); } else { - throw new Error('Could not find sort button'); + throw new Error("Could not find sort button"); } }); - console.log("Clicked sort dropdown button"); - // Wait for dropdown menu to appear await sleep(1000); // Find and click the appropriate sort option - const targetText = sortOrder === 'newest' ? 'Newest First' : 'Oldest First'; + const targetText = sortOrder === "newest" ? "Newest First" : "Oldest First"; // Wait for the dropdown menu and find the option by text await page.waitForFunction( (text) => { - const elements = Array.from(document.querySelectorAll('[role="menuitem"]')); - return elements.some(el => el.textContent?.includes(text)); + const elements = Array.from( + document.querySelectorAll('[role="menuitem"]'), + ); + return elements.some((el) => el.textContent?.includes(text)); }, { timeout: 5000 }, - targetText + targetText, ); // Click the option containing the target text await page.evaluate((text) => { - const elements = Array.from(document.querySelectorAll('[role="menuitem"]')); - const target = elements.find(el => el.textContent?.includes(text)); + const elements = Array.from( + document.querySelectorAll('[role="menuitem"]'), + ); + const target = elements.find((el) => el.textContent?.includes(text)); if (target) { (target as HTMLElement).click(); } else { @@ -61,12 +66,10 @@ async function setSortOrder(page: Page, sortOrder: 'newest' | 'oldest'): Promise } }, targetText); - console.log(`Clicked on "${targetText}" option`); // Wait for UI to update await sleep(2000); - console.log("Sort order has been set successfully"); } catch (error) { console.error(`Failed to set sort order: ${error}`); throw error; @@ -75,7 +78,7 @@ async function setSortOrder(page: Page, sortOrder: 'newest' | 'oldest'): Promise export async function scrollToBottomOfConversations( page: Page, - doneFile: DoneFile + doneFile: DoneFile, ): Promise { // Scroll to bottom and wait for more items until no new items load let previousHeight = 0; @@ -88,7 +91,7 @@ export async function scrollToBottomOfConversations( // Check if we've hit any processed URLs const foundProcessed = await page.evaluate((processedUrls) => { const items = Array.from( - document.querySelectorAll('div[data-testid="thread-title"]') + document.querySelectorAll('div[data-testid="thread-title"]'), ).map((div: Element) => div.closest("a") as HTMLAnchorElement); return items.some((item) => processedUrls.includes(item.href)); }, doneFile.processedUrls); @@ -120,7 +123,7 @@ export async function scrollToBottomOfConversations( export async function getConversations( page: Page, doneFile: DoneFile, - sortOrder: 'newest' | 'oldest' = 'newest' + sortOrder: "newest" | "oldest" = "newest", ): Promise { console.log("Navigating to library..."); await page.goto("https://www.perplexity.ai/library"); @@ -135,7 +138,7 @@ export async function getConversations( // Get all conversation links const conversations = await page.evaluate(() => { const items = Array.from( - document.querySelectorAll('div[data-testid^="thread-title-"]') + document.querySelectorAll('div[data-testid^="thread-title-"]'), ).map((div: Element) => div.closest("a") as HTMLAnchorElement); return items.map((item) => ({ title: item.textContent?.trim() || "Untitled", @@ -144,9 +147,11 @@ export async function getConversations( }); // Filter out already processed conversations - const filtered = conversations.filter((conv) => !doneFile.processedUrls.includes(conv.url)); + const filtered = conversations.filter( + (conv) => !doneFile.processedUrls.includes(conv.url), + ); // For "newest" sort order, reverse to process oldest first (as original behavior) // For "oldest" sort order, keep the order as-is since we want to process oldest first - return sortOrder === 'newest' ? filtered.reverse() : filtered; + return sortOrder === "newest" ? filtered.reverse() : filtered; }