Skip to content

Commit 14c6374

Browse files
ZengLiangYiZengLiangYi
committed
feat: add structured import contracts
1 parent c37e9cb commit 14c6374

10 files changed

Lines changed: 482 additions & 17 deletions

File tree

‎server/src/routes/notes-summarize-batch.test.ts‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
import assert from 'node:assert/strict';
2+
import { mkdtempSync, rmSync } from 'node:fs';
3+
import { tmpdir } from 'node:os';
4+
import { join } from 'node:path';
5+
import test from 'node:test';
6+
import Fastify from 'fastify';
7+
8+
const dataDir = mkdtempSync(join(tmpdir(), 'chatcrystal-summarize-route-test-'));
9+
process.env.DATA_DIR = dataDir;
10+
11+
const dbService = await import('../db/index.js');
12+
const queue = await import('../queue/index.js');
13+
const { noteRoutes } = await import('./notes.js');
14+
15+
function resetDatabase(db: Awaited<ReturnType<typeof dbService.initDatabase>>) {
16+
db.exec(`
17+
PRAGMA foreign_keys = ON;
18+
DELETE FROM experience_reviews;
19+
DELETE FROM note_tags;
20+
DELETE FROM embeddings;
21+
DELETE FROM note_relations;
22+
DELETE FROM notes;
23+
DELETE FROM messages;
24+
DELETE FROM conversations;
25+
DELETE FROM import_log;
26+
DELETE FROM vector_cleanup_tasks;
27+
`);
28+
}
29+
30+
function insertConversation(
31+
db: Awaited<ReturnType<typeof dbService.initDatabase>>,
32+
id: string,
33+
status: string,
34+
) {
35+
db.run(
36+
`INSERT INTO conversations (
37+
id, source, project_name, project_dir, message_count,
38+
first_message_at, last_message_at, file_path, file_size, file_mtime, status
39+
) VALUES (?, 'codex', 'p', 'p', 2, '2026-05-23', '2026-05-23', ?, 1, 'm', ?)`,
40+
[id, `C:/fixtures/${id}.jsonl`, status],
41+
);
42+
}
43+
44+
test('summarize by ids reports summarized and unknown conversations without queueing them', async () => {
45+
const db = await dbService.initDatabase();
46+
resetDatabase(db);
47+
insertConversation(db, 'old-id', 'summarized');
48+
49+
const app = Fastify();
50+
await app.register(noteRoutes);
51+
52+
const res = await app.inject({
53+
method: 'POST',
54+
url: '/api/summarize/batch-ids',
55+
payload: { conversationIds: ['old-id', 'old-id', 'missing-id'] },
56+
});
57+
58+
assert.equal(res.statusCode, 200);
59+
const body = res.json();
60+
assert.equal(body.success, true);
61+
assert.equal(body.data.queued, 0);
62+
assert.deepEqual(body.data.skipped, ['old-id']);
63+
assert.deepEqual(body.data.unknown, ['missing-id']);
64+
65+
await app.close();
66+
});
67+
68+
test('summarize by ids queues only requested imported conversations', async () => {
69+
const db = await dbService.initDatabase();
70+
resetDatabase(db);
71+
insertConversation(db, 'new-id', 'imported');
72+
insertConversation(db, 'old-id', 'summarized');
73+
74+
const app = Fastify();
75+
await app.register(noteRoutes);
76+
queue.summarizeQueue.pause();
77+
78+
try {
79+
const res = await app.inject({
80+
method: 'POST',
81+
url: '/api/summarize/batch-ids',
82+
payload: { conversationIds: ['new-id', 'old-id', 'missing-id'] },
83+
});
84+
85+
assert.equal(res.statusCode, 200);
86+
const body = res.json();
87+
assert.equal(body.success, true);
88+
assert.equal(body.data.queued, 1);
89+
assert.deepEqual(body.data.skipped, ['old-id']);
90+
assert.deepEqual(body.data.unknown, ['missing-id']);
91+
} finally {
92+
queue.cancelQueue();
93+
queue.summarizeQueue.start();
94+
}
95+
96+
await app.close();
97+
});
98+
99+
test('summarize by ids accepts empty bodies as empty requests', async () => {
100+
const app = Fastify();
101+
await app.register(noteRoutes);
102+
103+
const res = await app.inject({
104+
method: 'POST',
105+
url: '/api/summarize/batch-ids',
106+
});
107+
108+
assert.equal(res.statusCode, 200);
109+
const body = res.json();
110+
assert.equal(body.success, true);
111+
assert.equal(body.data.queued, 0);
112+
assert.deepEqual(body.data.skipped, []);
113+
assert.deepEqual(body.data.unknown, []);
114+
115+
await app.close();
116+
});
117+
118+
test('summarize status by ids returns conversation status or unknown', async () => {
119+
const db = await dbService.initDatabase();
120+
resetDatabase(db);
121+
insertConversation(db, 'new-id', 'imported');
122+
insertConversation(db, 'old-id', 'summarized');
123+
124+
const app = Fastify();
125+
await app.register(noteRoutes);
126+
127+
const res = await app.inject({
128+
method: 'POST',
129+
url: '/api/summarize/status-ids',
130+
payload: { conversationIds: ['new-id', 'old-id', 'missing-id'] },
131+
});
132+
133+
assert.equal(res.statusCode, 200);
134+
const body = res.json();
135+
assert.equal(body.success, true);
136+
assert.deepEqual(body.data.items, [
137+
{ id: 'new-id', status: 'imported' },
138+
{ id: 'old-id', status: 'summarized' },
139+
{ id: 'missing-id', status: 'unknown' },
140+
]);
141+
142+
await app.close();
143+
dbService.closeDatabase();
144+
rmSync(dataDir, { recursive: true, force: true });
145+
});

‎server/src/routes/notes.ts‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,64 @@ export async function noteRoutes(app: FastifyInstance) {
7676
};
7777
});
7878

79+
app.post('/api/summarize/batch-ids', async (req) => {
80+
const body = (req.body ?? {}) as { conversationIds?: unknown };
81+
const { conversationIds } = body;
82+
const requested = [...new Set(
83+
Array.isArray(conversationIds)
84+
? conversationIds.filter((id): id is string => typeof id === 'string' && id.trim().length > 0)
85+
: [],
86+
)];
87+
const db = getDatabase();
88+
const skipped: string[] = [];
89+
const unknown: string[] = [];
90+
let queued = 0;
91+
92+
for (const id of requested) {
93+
const row = db.exec(
94+
'SELECT project_name, slug, status FROM conversations WHERE id = ?',
95+
[id],
96+
)[0]?.values[0];
97+
if (!row) {
98+
unknown.push(id);
99+
continue;
100+
}
101+
102+
const [projectName, slug, status] = row as [string, string | null, string];
103+
if (status !== 'imported' || taskTracker.isTaskActive(id)) {
104+
skipped.push(id);
105+
continue;
106+
}
107+
108+
const title = `${projectName} / ${slug || id.slice(0, 8)}`;
109+
enqueueWithRetry(id, title, () => triggerSummarize(id)).catch((err) => {
110+
console.error(`[Summarize] Error for ${id}:`, err instanceof Error ? err.message : err);
111+
});
112+
queued++;
113+
}
114+
115+
return {
116+
success: true,
117+
data: { queued, skipped, unknown, queue: getQueueStatus() },
118+
};
119+
});
120+
121+
app.post('/api/summarize/status-ids', async (req) => {
122+
const body = (req.body ?? {}) as { conversationIds?: unknown };
123+
const { conversationIds } = body;
124+
const requested = [...new Set(
125+
Array.isArray(conversationIds)
126+
? conversationIds.filter((id): id is string => typeof id === 'string' && id.trim().length > 0)
127+
: [],
128+
)];
129+
const db = getDatabase();
130+
const items = requested.map((id) => {
131+
const row = db.exec('SELECT status FROM conversations WHERE id = ?', [id])[0]?.values[0];
132+
return { id, status: row ? String(row[0]) : 'unknown' };
133+
});
134+
return { success: true, data: { items, queue: getQueueStatus() } };
135+
});
136+
79137
// Reset error conversations back to 'imported' for retry
80138
app.post('/api/summarize/reset-errors', async () => {
81139
const db = getDatabase();

‎server/src/services/import.test.ts‎

Lines changed: 70 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,17 @@ process.env.DATA_DIR = mkdtempSync(join(tmpdir(), 'chatcrystal-import-test-'));
1616

1717
type ImportRuntime = {
1818
db: Database;
19-
importAll: () => Promise<{ imported: number; skipped: number; errors: number }>;
19+
importAll: () => Promise<{
20+
imported: number;
21+
replaced: number;
22+
skipped: number;
23+
errors: number;
24+
importedIds: string[];
25+
replacedIds: string[];
26+
skippedIds: string[];
27+
errorIds: string[];
28+
summarizationCandidateIds: string[];
29+
}>;
2030
registerAdapter: (adapter: SourceAdapter) => void;
2131
appConfig: { enabledSources: string[] };
2232
getUnsummarizedIds: () => string[];
@@ -310,7 +320,8 @@ test('importAll preserves user-rejected review links and gate state when reimpor
310320
);
311321
const cleanupTasks = vectorCleanupRows(db);
312322

313-
assert.equal(progress.imported, 1);
323+
assert.equal(progress.imported, 0);
324+
assert.equal(progress.replaced, 1);
314325
assert.deepEqual(conversation, [
315326
'filtered',
316327
86,
@@ -379,7 +390,8 @@ test('importAll resets ordinary changed conversations to imported and clears sta
379390
);
380391
const cleanupTasks = vectorCleanupRows(db);
381392

382-
assert.equal(progress.imported, 1);
393+
assert.equal(progress.imported, 0);
394+
assert.equal(progress.replaced, 1);
383395
assert.deepEqual(conversation, ['imported', null, null, null]);
384396
assert.deepEqual(messages, [
385397
'new ordinary user message',
@@ -438,6 +450,7 @@ test('importAll skips changed files when parsed content hash is unchanged', asyn
438450
const cleanupTasks = vectorCleanupRows(db);
439451

440452
assert.equal(progress.imported, 0);
453+
assert.equal(progress.replaced, 0);
441454
assert.equal(progress.skipped, 1);
442455
assert.deepEqual(conversation, [
443456
99,
@@ -450,6 +463,58 @@ test('importAll skips changed files when parsed content hash is unchanged', asyn
450463
assert.deepEqual(cleanupTasks, []);
451464
});
452465

466+
test('importAll returns structured ids and excludes skipped from summary candidates', async () => {
467+
const { db, importAll, registerAdapter, appConfig } = await loadRuntime();
468+
resetDatabase(db);
469+
470+
const source = 'test-structured-import-ids';
471+
const skippedParsed = parsedConversation('codex-same', source, [
472+
'hello',
473+
'world',
474+
]);
475+
const skippedHash = computeConversationContentHash(skippedParsed);
476+
477+
insertExistingConversation(db, {
478+
id: 'codex-same',
479+
source,
480+
status: 'imported',
481+
fileSize: 100,
482+
fileMtime: '2026-05-23T00:00:00.000Z',
483+
sourceConversationId: 'codex-same',
484+
contentHash: skippedHash,
485+
parserVersion: `${source}@test`,
486+
});
487+
488+
registerAdapter(
489+
testAdapter(
490+
source,
491+
[
492+
conversationMeta('codex-new', source, 100, '2026-05-23T00:00:00.000Z'),
493+
conversationMeta('codex-same', source, 100, '2026-05-23T00:00:00.000Z'),
494+
],
495+
new Map([
496+
[
497+
'codex-new',
498+
parsedConversation('codex-new', source, ['new user', 'new assistant']),
499+
],
500+
['codex-same', skippedParsed],
501+
]),
502+
),
503+
);
504+
appConfig.enabledSources = [source];
505+
506+
const result = await importAll();
507+
508+
assert.equal(result.imported, 1);
509+
assert.equal(result.replaced, 0);
510+
assert.equal(result.skipped, 1);
511+
assert.deepEqual(result.importedIds, ['codex-new']);
512+
assert.deepEqual(result.replacedIds, []);
513+
assert.deepEqual(result.skippedIds, ['codex-same']);
514+
assert.deepEqual(result.errorIds, []);
515+
assert.deepEqual(result.summarizationCandidateIds, ['codex-new']);
516+
});
517+
453518
test('importAll preserves edited notes when replacing changed local conversations', async () => {
454519
const { db, importAll, registerAdapter, appConfig, getUnsummarizedIds } =
455520
await loadRuntime();
@@ -503,7 +568,8 @@ test('importAll preserves edited notes when replacing changed local conversation
503568
)[0].values[0];
504569
const cleanupTasks = vectorCleanupRows(db);
505570

506-
assert.equal(progress.imported, 1);
571+
assert.equal(progress.imported, 0);
572+
assert.equal(progress.replaced, 1);
507573
assert.deepEqual(messages, [
508574
'new edited-note user message',
509575
'new edited-note assistant message',

0 commit comments

Comments
 (0)