Skip to content

Commit 8086439

Browse files
authored
webui: export conversations as jsonl (#24688)
* webui: export conversations as jsonl each session is one jsonl file, a session header line followed by one line per message exporting multiple conversations bundles them into a zip, one jsonl file each * webui: import jsonl and zip conversation exports parse the new jsonl session format and zip archives on import keep supporting the legacy json format
1 parent 558e221 commit 8086439

5 files changed

Lines changed: 190 additions & 45 deletions

File tree

tools/ui/package-lock.json

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

tools/ui/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
"eslint-config-prettier": "10.1.8",
6060
"eslint-plugin-storybook": "10.4.2",
6161
"eslint-plugin-svelte": "3.19.0",
62+
"fflate": "0.8.3",
6263
"globals": "16.5.0",
6364
"highlight.js": "11.11.1",
6465
"http-server": "14.1.1",

tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte

Lines changed: 14 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -132,14 +132,18 @@
132132
133133
async function handleExportConfirm(selectedConversations: DatabaseConversation[]) {
134134
try {
135-
const allData: ExportedConversations = await Promise.all(
135+
const allData: ExportedConversation[] = await Promise.all(
136136
selectedConversations.map(async (conv) => {
137137
const messages = await conversationsStore.getConversationMessages(conv.id);
138138
return { conv: $state.snapshot(conv), messages: $state.snapshot(messages) };
139139
})
140140
);
141141
142-
conversationsStore.downloadConversationFile(allData);
142+
if (allData.length === 1) {
143+
conversationsStore.downloadConversationFile(allData[0]);
144+
} else {
145+
conversationsStore.downloadConversationsArchive(allData);
146+
}
143147
144148
exportedConversations = selectedConversations;
145149
showExportSummary = true;
@@ -156,37 +160,21 @@
156160
const input = document.createElement('input');
157161
158162
input.type = HtmlInputType.FILE;
159-
input.accept = FileExtensionText.JSON;
163+
input.accept = `${FileExtensionText.JSON},${FileExtensionText.JSONL},${FileExtensionText.ZIP}`;
160164
161165
input.onchange = async (e) => {
162166
const file = (e.target as HTMLInputElement)?.files?.[0];
163167
if (!file) return;
164168
165169
try {
166-
const text = await file.text();
167-
const parsedData = JSON.parse(text);
168-
let importedData: ExportedConversations;
169-
170-
if (Array.isArray(parsedData)) {
171-
importedData = parsedData;
172-
} else if (
173-
parsedData &&
174-
typeof parsedData === 'object' &&
175-
'conv' in parsedData &&
176-
'messages' in parsedData
177-
) {
178-
// Single conversation object
179-
importedData = [parsedData];
180-
} else {
181-
throw new Error(
182-
'Invalid file format: expected array of conversations or single conversation object'
183-
);
170+
const importedData = await conversationsStore.parseImportFile(file);
171+
172+
if (importedData.length === 0) {
173+
throw new Error('No conversations found in file');
184174
}
185175
186176
fullImportData = importedData;
187-
availableConversations = importedData.map(
188-
(item: { conv: DatabaseConversation; messages: DatabaseMessage[] }) => item.conv
189-
);
177+
availableConversations = importedData.map((item) => item.conv);
190178
messageCountMap = createMessageCountMap(importedData);
191179
showImportDialog = true;
192180
} catch (err: unknown) {
@@ -258,7 +246,7 @@
258246
<SettingsGroup title="Conversations">
259247
<SettingsChatImportExportSection
260248
title="Export"
261-
description="Download your conversations as a JSON file. This includes all messages, attachments, and conversation history."
249+
description="Download your conversations as a ZIP of JSONL files. This includes all messages, attachments, and conversation history."
262250
IconComponent={Download}
263251
buttonText="Export conversations"
264252
onclick={handleExportClick}
@@ -267,7 +255,7 @@
267255

268256
<SettingsChatImportExportSection
269257
title="Import"
270-
description="Import one or more conversations from a previously exported JSON file. This will merge with your existing conversations."
258+
description="Import one or more conversations from a previously exported ZIP or JSONL file. This will merge with your existing conversations."
271259
IconComponent={Upload}
272260
buttonText="Import conversations"
273261
onclick={handleImportClick}

tools/ui/src/lib/enums/files.enums.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,8 @@ export enum FileExtensionText {
123123
HTML = '.html',
124124
HTM = '.htm',
125125
JSON = '.json',
126+
JSONL = '.jsonl',
127+
ZIP = '.zip',
126128
XML = '.xml',
127129
YAML = '.yaml',
128130
YML = '.yml',
@@ -179,7 +181,8 @@ export enum UriPattern {
179181
// MIME type enums
180182
export enum MimeTypeApplication {
181183
PDF = 'application/pdf',
182-
OCTET_STREAM = 'application/octet-stream'
184+
OCTET_STREAM = 'application/octet-stream',
185+
ZIP = 'application/zip'
183186
}
184187

185188
export enum MimeTypeAudio {
@@ -226,6 +229,7 @@ export enum MimeTypeText {
226229
CSS = 'text/css',
227230
HTML = 'text/html',
228231
JSON = 'application/json',
232+
JSONL = 'application/jsonl',
229233
XML_TEXT = 'text/xml',
230234
XML_APP = 'application/xml',
231235
YAML_TEXT = 'text/yaml',

tools/ui/src/lib/stores/conversations.svelte.ts

Lines changed: 162 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,15 @@ import { MigrationService } from '$lib/services/migration.service';
2626
import { config } from '$lib/stores/settings.svelte';
2727
import { filterByLeafNodeId, findLeafNode, generateConversationTitle } from '$lib/utils';
2828
import type { McpServerOverride } from '$lib/types/database';
29-
import { MessageRole, HtmlInputType, FileExtensionText, ReasoningEffort } from '$lib/enums';
29+
import { zipSync, unzipSync, strToU8, strFromU8 } from 'fflate';
30+
import {
31+
MessageRole,
32+
HtmlInputType,
33+
FileExtensionText,
34+
MimeTypeText,
35+
MimeTypeApplication,
36+
ReasoningEffort
37+
} from '$lib/enums';
3038
import {
3139
ISO_DATE_TIME_SEPARATOR,
3240
ISO_DATE_TIME_SEPARATOR_REPLACEMENT,
@@ -934,41 +942,177 @@ class ConversationsStore {
934942
.replace(ISO_DATE_TIME_SEPARATOR, ISO_DATE_TIME_SEPARATOR_REPLACEMENT)
935943
.replaceAll(ISO_TIME_SEPARATOR, ISO_TIME_SEPARATOR_REPLACEMENT);
936944
const trimmedConvId = conversation.id?.slice(0, EXPORT_CONV_ID_TRIM_LENGTH) ?? '';
937-
return `${formattedDate}_conv_${trimmedConvId}_${sanitizedName}.json`;
945+
return `${formattedDate}_conv_${trimmedConvId}_${sanitizedName}${FileExtensionText.JSONL}`;
946+
}
947+
948+
/**
949+
* Serializes a session (a conversation with its messages) as JSONL.
950+
* The first line is the session header (a `type: 'session'` record carrying the
951+
* conversation properties); each subsequent line is a single message.
952+
* @param data - The exported conversation payload
953+
* @returns The JSONL string (one record per line)
954+
*/
955+
serializeSessionToJsonl(data: ExportedConversation): string {
956+
const { conv, messages } = data;
957+
958+
const sessionLine = JSON.stringify({ type: 'session', harness: 'llama.app', ...conv });
959+
const messageLines = messages.map((message: DatabaseMessage) => {
960+
// `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it.
961+
const { toolCalls, ...rest } = message;
962+
const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest;
963+
964+
return JSON.stringify({ type: 'message', message: normalized });
965+
});
966+
967+
return [sessionLine, ...messageLines].join('\n');
968+
}
969+
970+
/**
971+
* Parses the JSONL session format produced by {@link serializeSessionToJsonl}.
972+
* A `type: 'session'` line starts a new session; following `type: 'message'`
973+
* lines are appended to it. Supports multiple sessions in a single file.
974+
* @param text - The JSONL file contents
975+
* @returns The parsed conversations with their messages
976+
*/
977+
parseSessionsJsonl(text: string): ExportedConversation[] {
978+
const sessions: ExportedConversation[] = [];
979+
let current: ExportedConversation | null = null;
980+
981+
for (const line of text.split('\n')) {
982+
const trimmed = line.trim();
983+
if (!trimmed) continue;
984+
985+
const record = JSON.parse(trimmed);
986+
987+
if (record.type === 'session') {
988+
// Drop the discriminator and harness marker; the rest is the conversation.
989+
const conv = { ...record };
990+
delete conv.type;
991+
delete conv.harness;
992+
current = { conv: conv as DatabaseConversation, messages: [] };
993+
sessions.push(current);
994+
} else if (record.type === 'message') {
995+
if (!current) {
996+
throw new Error('Invalid JSONL: message record before any session record');
997+
}
998+
999+
const message = record.message as DatabaseMessage;
1000+
// `toolCalls` is parsed to an array on export; the DB stores it as a string.
1001+
if (message.toolCalls !== undefined && typeof message.toolCalls !== 'string') {
1002+
message.toolCalls = JSON.stringify(message.toolCalls);
1003+
}
1004+
current.messages.push(message);
1005+
}
1006+
// Ignore unknown record types for forward compatibility.
1007+
}
1008+
1009+
return sessions;
1010+
}
1011+
1012+
/**
1013+
* Parses an import file into conversations, accepting the current `.jsonl` and
1014+
* `.zip` formats as well as the legacy `.json` format.
1015+
* @param file - The user-selected file
1016+
* @returns The parsed conversations with their messages
1017+
*/
1018+
async parseImportFile(file: File): Promise<ExportedConversation[]> {
1019+
const name = file.name.toLowerCase();
1020+
1021+
if (name.endsWith(FileExtensionText.ZIP)) {
1022+
const entries = unzipSync(new Uint8Array(await file.arrayBuffer()));
1023+
const sessions: ExportedConversation[] = [];
1024+
for (const [entryName, bytes] of Object.entries(entries)) {
1025+
if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue;
1026+
sessions.push(...this.parseSessionsJsonl(strFromU8(bytes)));
1027+
}
1028+
return sessions;
1029+
}
1030+
1031+
const text = await file.text();
1032+
1033+
if (name.endsWith(FileExtensionText.JSONL)) {
1034+
return this.parseSessionsJsonl(text);
1035+
}
1036+
1037+
// Legacy JSON format: an array of conversations or a single conversation object.
1038+
const parsed = JSON.parse(text);
1039+
if (Array.isArray(parsed)) {
1040+
return parsed;
1041+
}
1042+
if (parsed && typeof parsed === 'object' && 'conv' in parsed && 'messages' in parsed) {
1043+
return [parsed];
1044+
}
1045+
throw new Error(
1046+
'Invalid file format: expected array of conversations or single conversation object'
1047+
);
9381048
}
9391049

9401050
/**
9411051
* Triggers a browser download of the provided exported conversation data
942-
* @param data - The exported conversation payload (either a single conversation or array of them)
1052+
* @param data - The exported conversation payload (a single conversation with its messages)
9431053
* @param filename - Filename; if omitted, a deterministic name is generated
9441054
*/
945-
downloadConversationFile(data: ExportedConversations, filename?: string): void {
946-
// Choose the first conversation or message
947-
const conversation =
948-
'conv' in data ? data.conv : Array.isArray(data) ? data[0]?.conv : undefined;
949-
const msgs =
950-
'messages' in data ? data.messages : Array.isArray(data) ? data[0]?.messages : undefined;
1055+
downloadConversationFile(data: ExportedConversation, filename?: string): void {
1056+
const { conv: conversation, messages: msgs } = data;
9511057

9521058
if (!conversation) {
9531059
console.error('Invalid data: missing conversation');
9541060
return;
9551061
}
9561062

957-
let downloadFilename: string;
1063+
const downloadFilename = filename ?? this.generateConversationFilename(conversation, msgs);
9581064

959-
if (filename) {
960-
downloadFilename = filename;
961-
} else if (Array.isArray(data) && data.length > 1) {
962-
downloadFilename = `${new Date().toISOString().split(ISO_DATE_TIME_SEPARATOR)[0]}_conversations.json`;
963-
} else {
964-
downloadFilename = this.generateConversationFilename(conversation, msgs);
1065+
const jsonl = this.serializeSessionToJsonl(data);
1066+
const blob = new Blob([jsonl], { type: MimeTypeText.JSONL });
1067+
this.triggerDownload(blob, downloadFilename);
1068+
}
1069+
1070+
/**
1071+
* Triggers a browser download of multiple conversations as a `.zip`, one
1072+
* `.jsonl` file per conversation.
1073+
* @param data - The conversations to export
1074+
*/
1075+
downloadConversationsArchive(data: ExportedConversation[]): void {
1076+
if (data.length === 0) {
1077+
console.error('Invalid data: no conversations to export');
1078+
return;
9651079
}
9661080

967-
const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });
1081+
const usedNames = new SvelteSet<string>();
1082+
const files: Record<string, Uint8Array> = {};
1083+
1084+
for (const session of data) {
1085+
const baseName = this.generateConversationFilename(session.conv, session.messages);
1086+
1087+
// Disambiguate any duplicate filenames within the archive.
1088+
let entryName = baseName;
1089+
let suffix = 1;
1090+
while (usedNames.has(entryName)) {
1091+
entryName = baseName.replace(
1092+
new RegExp(`${FileExtensionText.JSONL}$`),
1093+
`_${suffix++}${FileExtensionText.JSONL}`
1094+
);
1095+
}
1096+
usedNames.add(entryName);
1097+
1098+
files[entryName] = strToU8(this.serializeSessionToJsonl(session));
1099+
}
1100+
1101+
const archiveName = `${new Date().toISOString().split(ISO_DATE_TIME_SEPARATOR)[0]}_conversations${FileExtensionText.ZIP}`;
1102+
1103+
const zipped = zipSync(files);
1104+
const blob = new Blob([zipped], { type: MimeTypeApplication.ZIP });
1105+
this.triggerDownload(blob, archiveName);
1106+
}
1107+
1108+
/**
1109+
* Triggers a browser download of a blob under the given filename.
1110+
*/
1111+
private triggerDownload(blob: Blob, filename: string): void {
9681112
const url = URL.createObjectURL(blob);
9691113
const a = document.createElement('a');
9701114
a.href = url;
971-
a.download = downloadFilename;
1115+
a.download = filename;
9721116
document.body.appendChild(a);
9731117
a.click();
9741118
document.body.removeChild(a);

0 commit comments

Comments
 (0)