Skip to content

Commit 28b637e

Browse files
SimplyLizSimplyLiz
committed
feat(compression): add embedSummaryId, forceConverge, and dedup target IDs
- embedSummaryId option embeds summary_id in content as [summary#id: ...] so downstream tools can reference compressed summaries - forceConverge option hard-truncates non-recency messages to 512 chars when binary search bottoms out, guaranteeing convergence under budget - Dedup tags now include the keep-target message ID for debuggability: [uc:dup of msgId — N chars] instead of [uc:dup — N chars, see later message] - Updated prefix checks in classifyAll and dedup to handle [summary#, [truncated formats for re-compression idempotency
1 parent 5a1736b commit 28b637e

4 files changed

Lines changed: 395 additions & 34 deletions

File tree

‎src/compress.ts‎

Lines changed: 128 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -363,12 +363,14 @@ function formatSummary(
363363
rawText: string,
364364
mergeCount?: number,
365365
skipEntities?: boolean,
366+
summaryId?: string,
366367
): string {
367368
const entitySuffix = skipEntities
368369
? ''
369370
: (() => { const e = extractEntities(rawText); return e.length > 0 ? ` | entities: ${e.join(', ')}` : ''; })();
370371
const mergeSuffix = mergeCount && mergeCount > 1 ? ` (${mergeCount} messages merged)` : '';
371-
return `[summary: ${summaryText}${mergeSuffix}${entitySuffix}]`;
372+
const prefix = summaryId ? `[summary#${summaryId}: ` : '[summary: ';
373+
return `${prefix}${summaryText}${mergeSuffix}${entitySuffix}]`;
372374
}
373375

374376
/** Collect consecutive non-preserved, non-codeSplit, non-dedup messages with the same role. */
@@ -418,7 +420,7 @@ function classifyAll(
418420
if (content.length < 120) {
419421
return { msg, preserved: true };
420422
}
421-
if (content.startsWith('[summary:')) {
423+
if (content.startsWith('[summary:') || content.startsWith('[summary#') || content.startsWith('[truncated')) {
422424
return { msg, preserved: true };
423425
}
424426
if (dedupAnnotations?.has(idx)) {
@@ -549,9 +551,10 @@ function compressSync(
549551
// Dedup: replace earlier duplicate/near-duplicate with compact reference
550552
if (classified[i].dedup) {
551553
const annotation = classified[i].dedup!;
554+
const keepTargetId = messages[annotation.duplicateOfIndex].id;
552555
const tag = annotation.similarity != null
553-
? `[uc:near-dup — ${annotation.contentLength} chars, ~${Math.round(annotation.similarity * 100)}% match, see later message]`
554-
: `[uc:dup — ${annotation.contentLength} chars, see later message]`;
556+
? `[uc:near-dup of ${keepTargetId} — ${annotation.contentLength} chars, ~${Math.round(annotation.similarity * 100)}% match]`
557+
: `[uc:dup of ${keepTargetId} — ${annotation.contentLength} chars]`;
555558
result.push(buildCompressedMessage(msg, [msg.id], tag, sourceVersion, verbatim, [msg]));
556559
if (annotation.similarity != null) {
557560
messagesFuzzyDeduped++;
@@ -570,7 +573,8 @@ function compressSync(
570573
const codeFences = segments.filter(s => s.type === 'code').map(s => s.content);
571574
const proseBudget = proseText.length < 600 ? 200 : 400;
572575
const summaryText = summarize(proseText, proseBudget);
573-
const compressed = `${formatSummary(summaryText, proseText, undefined, true)}\n\n${codeFences.join('\n\n')}`;
576+
const embeddedId = options.embedSummaryId ? makeSummaryId([msg.id]) : undefined;
577+
const compressed = `${formatSummary(summaryText, proseText, undefined, true, embeddedId)}\n\n${codeFences.join('\n\n')}`;
574578

575579
if (compressed.length >= content.length) {
576580
result.push(msg);
@@ -596,10 +600,12 @@ function compressSync(
596600
: summarize(allContent, contentBudget);
597601

598602
if (group.length > 1) {
599-
let summary = formatSummary(summaryText, allContent, group.length);
603+
const mergeIds = group.map(g => g.msg.id);
604+
const embeddedId = options.embedSummaryId ? makeSummaryId(mergeIds) : undefined;
605+
let summary = formatSummary(summaryText, allContent, group.length, undefined, embeddedId);
600606
const combinedLength = group.reduce((sum, g) => sum + contentLength(g.msg), 0);
601607
if (summary.length >= combinedLength) {
602-
summary = formatSummary(summaryText, allContent, group.length, true);
608+
summary = formatSummary(summaryText, allContent, group.length, true, embeddedId);
603609
}
604610

605611
if (summary.length >= combinedLength) {
@@ -609,17 +615,17 @@ function compressSync(
609615
}
610616
} else {
611617
const sourceMsgs = group.map(g => g.msg);
612-
const mergeIds = sourceMsgs.map(m => m.id);
613618
const base: Message = { ...sourceMsgs[0] };
614619
result.push(buildCompressedMessage(base, mergeIds, summary, sourceVersion, verbatim, sourceMsgs));
615620
messagesCompressed += group.length;
616621
}
617622
} else {
618623
const single = group[0].msg;
619624
const content = typeof single.content === 'string' ? single.content : '';
620-
let summary = formatSummary(summaryText, allContent);
625+
const embeddedId = options.embedSummaryId ? makeSummaryId([single.id]) : undefined;
626+
let summary = formatSummary(summaryText, allContent, undefined, undefined, embeddedId);
621627
if (summary.length >= content.length) {
622-
summary = formatSummary(summaryText, allContent, undefined, true);
628+
summary = formatSummary(summaryText, allContent, undefined, true, embeddedId);
623629
}
624630

625631
if (summary.length >= content.length) {
@@ -718,9 +724,10 @@ async function compressAsync(
718724
// Dedup: replace earlier duplicate/near-duplicate with compact reference
719725
if (classified[i].dedup) {
720726
const annotation = classified[i].dedup!;
727+
const keepTargetId = messages[annotation.duplicateOfIndex].id;
721728
const tag = annotation.similarity != null
722-
? `[uc:near-dup — ${annotation.contentLength} chars, ~${Math.round(annotation.similarity * 100)}% match, see later message]`
723-
: `[uc:dup — ${annotation.contentLength} chars, see later message]`;
729+
? `[uc:near-dup of ${keepTargetId} — ${annotation.contentLength} chars, ~${Math.round(annotation.similarity * 100)}% match]`
730+
: `[uc:dup of ${keepTargetId} — ${annotation.contentLength} chars]`;
724731
result.push(buildCompressedMessage(msg, [msg.id], tag, sourceVersion, verbatim, [msg]));
725732
if (annotation.similarity != null) {
726733
messagesFuzzyDeduped++;
@@ -739,7 +746,8 @@ async function compressAsync(
739746
const codeFences = segments.filter(s => s.type === 'code').map(s => s.content);
740747
const proseBudget = proseText.length < 600 ? 200 : 400;
741748
const summaryText = await withFallback(proseText, userSummarizer, proseBudget);
742-
const compressed = `${formatSummary(summaryText, proseText, undefined, true)}\n\n${codeFences.join('\n\n')}`;
749+
const embeddedId = options.embedSummaryId ? makeSummaryId([msg.id]) : undefined;
750+
const compressed = `${formatSummary(summaryText, proseText, undefined, true, embeddedId)}\n\n${codeFences.join('\n\n')}`;
743751

744752
if (compressed.length >= content.length) {
745753
result.push(msg);
@@ -765,10 +773,12 @@ async function compressAsync(
765773
: await withFallback(allContent, userSummarizer, contentBudget);
766774

767775
if (group.length > 1) {
768-
let summary = formatSummary(summaryText, allContent, group.length);
776+
const mergeIds = group.map(g => g.msg.id);
777+
const embeddedId = options.embedSummaryId ? makeSummaryId(mergeIds) : undefined;
778+
let summary = formatSummary(summaryText, allContent, group.length, undefined, embeddedId);
769779
const combinedLength = group.reduce((sum, g) => sum + contentLength(g.msg), 0);
770780
if (summary.length >= combinedLength) {
771-
summary = formatSummary(summaryText, allContent, group.length, true);
781+
summary = formatSummary(summaryText, allContent, group.length, true, embeddedId);
772782
}
773783

774784
if (summary.length >= combinedLength) {
@@ -778,17 +788,17 @@ async function compressAsync(
778788
}
779789
} else {
780790
const sourceMsgs = group.map(g => g.msg);
781-
const mergeIds = sourceMsgs.map(m => m.id);
782791
const base: Message = { ...sourceMsgs[0] };
783792
result.push(buildCompressedMessage(base, mergeIds, summary, sourceVersion, verbatim, sourceMsgs));
784793
messagesCompressed += group.length;
785794
}
786795
} else {
787796
const single = group[0].msg;
788797
const content = typeof single.content === 'string' ? single.content : '';
789-
let summary = formatSummary(summaryText, allContent);
798+
const embeddedId = options.embedSummaryId ? makeSummaryId([single.id]) : undefined;
799+
let summary = formatSummary(summaryText, allContent, undefined, undefined, embeddedId);
790800
if (summary.length >= content.length) {
791-
summary = formatSummary(summaryText, allContent, undefined, true);
801+
summary = formatSummary(summaryText, allContent, undefined, true, embeddedId);
792802
}
793803

794804
if (summary.length >= content.length) {
@@ -846,6 +856,80 @@ function addBudgetFields(cr: CompressResult, tokenBudget: number, recencyWindow:
846856
return { ...cr, fits: tokens <= tokenBudget, tokenCount: tokens, recencyWindow };
847857
}
848858

859+
/**
860+
* Force-converge pass: hard-truncate non-recency messages to guarantee the
861+
* result fits within the token budget. Mirrors LCM Level 3 DeterministicTruncate.
862+
*/
863+
function forceConvergePass(
864+
cr: CompressResult,
865+
tokenBudget: number,
866+
preserveRoles: Set<string>,
867+
sourceVersion: number,
868+
): CompressResult {
869+
if (cr.fits) return cr;
870+
871+
const recencyWindow = cr.recencyWindow ?? 0;
872+
const cutoff = Math.max(0, cr.messages.length - recencyWindow);
873+
874+
// Collect eligible messages: before recency cutoff, not in preserveRoles, content > 512 chars
875+
type Candidate = { idx: number; contentLen: number };
876+
const candidates: Candidate[] = [];
877+
878+
for (let i = 0; i < cutoff; i++) {
879+
const m = cr.messages[i];
880+
const content = typeof m.content === 'string' ? m.content : '';
881+
if (m.role && preserveRoles.has(m.role)) continue;
882+
if (content.length <= 512) continue;
883+
candidates.push({ idx: i, contentLen: content.length });
884+
}
885+
886+
// Sort by content length descending (biggest savings first)
887+
candidates.sort((a, b) => b.contentLen - a.contentLen);
888+
889+
// Clone messages and verbatim for mutation
890+
const messages = cr.messages.map(m => ({ ...m, metadata: m.metadata ? { ...m.metadata } : {} }));
891+
const verbatim = { ...cr.verbatim };
892+
let tokenCount = cr.tokenCount!;
893+
894+
for (const cand of candidates) {
895+
if (tokenCount <= tokenBudget) break;
896+
897+
const m = messages[cand.idx];
898+
const content = typeof m.content === 'string' ? m.content : '';
899+
const truncated = content.slice(0, 512);
900+
const tag = `[truncated — ${content.length} chars: ${truncated}]`;
901+
902+
const oldTokens = estimateTokens(m);
903+
904+
// If already compressed (has _uc_original), just replace content in-place
905+
const hasOriginal = !!(m.metadata?._uc_original);
906+
if (hasOriginal) {
907+
messages[cand.idx] = { ...m, content: tag };
908+
} else {
909+
// Store original in verbatim and add provenance
910+
verbatim[m.id] = { ...m };
911+
messages[cand.idx] = {
912+
...m,
913+
content: tag,
914+
metadata: {
915+
...(m.metadata ?? {}),
916+
_uc_original: {
917+
ids: [m.id],
918+
summary_id: makeSummaryId([m.id]),
919+
version: sourceVersion,
920+
},
921+
},
922+
};
923+
}
924+
925+
const newTokens = estimateTokens(messages[cand.idx]);
926+
tokenCount -= (oldTokens - newTokens);
927+
}
928+
929+
const fits = tokenCount <= tokenBudget;
930+
return { ...cr, messages, verbatim, fits, tokenCount };
931+
}
932+
849933
function compressSyncWithBudget(
850934
messages: Message[],
851935
tokenBudget: number,
@@ -875,10 +959,20 @@ function compressSyncWithBudget(
875959
}
876960
}
877961

878-
if (lastRw === lo && lastResult) return lastResult;
962+
let result: CompressResult;
963+
if (lastRw === lo && lastResult) {
964+
result = lastResult;
965+
} else {
966+
const cr = compressSync(messages, { ...options, recencyWindow: lo, summarizer: undefined, tokenBudget: undefined });
967+
result = addBudgetFields(cr, tokenBudget, lo);
968+
}
879969

880-
const cr = compressSync(messages, { ...options, recencyWindow: lo, summarizer: undefined, tokenBudget: undefined });
881-
return addBudgetFields(cr, tokenBudget, lo);
970+
if (!result.fits && options.forceConverge) {
971+
const preserveRoles = new Set(options.preserve ?? ['system']);
972+
result = forceConvergePass(result, tokenBudget, preserveRoles, sourceVersion);
973+
}
974+
975+
return result;
882976
}
883977

884978
async function compressAsyncWithBudget(
@@ -910,10 +1004,20 @@ async function compressAsyncWithBudget(
9101004
}
9111005
}
9121006

913-
if (lastRw === lo && lastResult) return lastResult;
1007+
let result: CompressResult;
1008+
if (lastRw === lo && lastResult) {
1009+
result = lastResult;
1010+
} else {
1011+
const cr = await compressAsync(messages, { ...options, recencyWindow: lo, tokenBudget: undefined });
1012+
result = addBudgetFields(cr, tokenBudget, lo);
1013+
}
9141014

915-
const cr = await compressAsync(messages, { ...options, recencyWindow: lo, tokenBudget: undefined });
916-
return addBudgetFields(cr, tokenBudget, lo);
1015+
if (!result.fits && options.forceConverge) {
1016+
const preserveRoles = new Set(options.preserve ?? ['system']);
1017+
result = forceConvergePass(result, tokenBudget, preserveRoles, sourceVersion);
1018+
}
1019+
1020+
return result;
9171021
}
9181022

9191023
// ---------------------------------------------------------------------------

‎src/dedup.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ export function analyzeDuplicates(
3131
// Skip ineligible messages
3232
if (msg.role && preserveRoles.has(msg.role)) continue;
3333
if (msg.tool_calls && Array.isArray(msg.tool_calls) && msg.tool_calls.length > 0) continue;
34-
if (content.startsWith('[summary:')) continue;
34+
if (content.startsWith('[summary:') || content.startsWith('[summary#') || content.startsWith('[truncated')) continue;
3535
if (content.length < 200) continue;
3636

3737
const hash = djb2(content);
@@ -121,7 +121,7 @@ export function analyzeFuzzyDuplicates(
121121
// Same skip criteria as analyzeDuplicates
122122
if (msg.role && preserveRoles.has(msg.role)) continue;
123123
if (msg.tool_calls && Array.isArray(msg.tool_calls) && msg.tool_calls.length > 0) continue;
124-
if (content.startsWith('[summary:')) continue;
124+
if (content.startsWith('[summary:') || content.startsWith('[summary#') || content.startsWith('[truncated')) continue;
125125
if (content.length < 200) continue;
126126

127127
// Skip indices already handled by exact dedup

‎src/types.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@ export type CompressOptions = {
2828
fuzzyDedup?: boolean;
2929
/** Similarity threshold for fuzzy dedup (0-1). Default: 0.85. */
3030
fuzzyThreshold?: number;
31+
/** Embed summary_id in compressed content so downstream tools can reference it. Default: false. */
32+
embedSummaryId?: boolean;
33+
/** Hard-truncate non-recency messages when binary search bottoms out and budget still exceeded. Default: false. */
34+
forceConverge?: boolean;
3135
};
3236

3337
export type VerbatimMap = Record<string, Message>;

0 commit comments

Comments
 (0)