Skip to content

Commit f2e290f

Browse files
author
Tehan
committed
fix(rebase): complete v0.26 resolution — SQL SET comma, dup pi import, two-phase promotion test, config-docs regen
- storage-meta-shared.ts: add missing comma after cached_m0_project_identity in the persistCachedM0 SET clause (TAKE-BOTH merge dropped the separator). - inject-compartments-pi.ts: drop duplicate renderExternalMemoryBlock import (026b856 added it standalone; 2a8cd71 added it to the consolidated block). - promotion.test.ts: migrate the 3 external-tee tests from the dropped promoteSessionFactsToMemory to upstream's two-phase promoteSessionFactsDurable + embedPromotedFacts (tee now lives post-commit). - configuration.md: regenerate to include memory.external.* rows. - lint:fix import ordering.
1 parent dcd2227 commit f2e290f

10 files changed

Lines changed: 188 additions & 167 deletions

File tree

‎packages/docs/src/content/docs/reference/configuration.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,24 @@ Durable project memory, semantic search, and recall features.
105105
| `memory.git_commit_indexing.enabled` | boolean | `false` | Index HEAD git commits for ctx_search (git_commit source). Graduated from experimental.git_commit_indexing; opt-in, default off. Independent of memory.enabled. |
106106
| `memory.git_commit_indexing.since_days` | number (7–3650) | `365` | Days of HEAD history to index (min: 7, max: 3650, default: 365) |
107107
| `memory.git_commit_indexing.max_commits` | number (100–20000) | `2000` | Max commits kept per project; oldest evicted (min: 100, max: 20000, default: 2000) |
108+
| `memory.external` | object | — | External long-term memory backend (tee). USER config only. |
109+
| `memory.external.provider` | `"hindsight"` \\| `"off"` | `"off"` | External memory backend. 'hindsight' tees memory creations to a Hindsight service; 'off' disables (default). SECURITY: this whole block only honors USER-level config. |
110+
| `memory.external.endpoint` | string | — | Backend base URL (e.g. http://10.0.0.1:8889). Required when provider is hindsight. |
111+
| `memory.external.api_key` | string | — | Bearer token for the backend (optional). |
112+
| `memory.external.project_bank` | string | `"mc-{name}-{id8}"` | Bank name template for project-scoped items. Placeholders: {name}=project basename, {id8}=first 8 chars of the project identity hash. |
113+
| `memory.external.main_bank` | string | — | Bank for user- and global-scoped items. Required when provider is hindsight. Assumed to pre-exist; never created or modified. |
114+
| `memory.external.retain_sources` | `"historian"` \\| `"agent"` \\| `"dreamer"`[] | `["historian","agent","dreamer"]` | Which creation points tee: historian promotion, agent ctx_memory writes, dreamer user-memory promotion. |
115+
| `memory.external.tags` | string[] | `[]` | Static tags attached to every retained item. |
116+
| `memory.external.recall` | object | — | Unified read path: session-start recall + ctx_search external source. |
117+
| `memory.external.recall.enabled` | boolean | `true` | Session-start recall from the external backend, merged into the context injection (default: true). |
118+
| `memory.external.recall.timeout_ms` | number (500–15000) | `3000` | Max wait for recall at the first render of a session (already cache-cold). Late results ride the m[1] delta. (default: 3000) |
119+
| `memory.external.recall.max_tokens` | number (256–8192) | `2048` | Token budget per recall slice (project / profile / global each). (default: 2048) |
120+
| `memory.external.recall.dedup_threshold` | number (0.5–0.99) | `0.85` | Cosine similarity above which a recalled item is dropped as a duplicate of a local memory. Hash-only fallback when embeddings are unavailable. (default: 0.85) |
121+
| `memory.external.recall.global_tags` | string[] | `[]` | Tag filter for the global (main-bank) recall slice, matched with tags_match 'any' (untagged content INCLUDED). Empty = no filter (full autoRecall replacement). |
122+
| `memory.external.recall.global_from_prompt` | boolean | `false` | Include an excerpt of the session's FIRST user prompt in the global-slice recall query (the project name is always included). The first prompt is fixed for the session, so the query — and the frozen recall snapshot — stays deterministic. Default false: pure template query. |
123+
| `memory.external.recall.search` | boolean | `true` | Expose the ctx_search 'external' source (project + main bank). (default: true) |
124+
| `memory.external.recall.mental_models` | boolean | `true` | Use Hindsight mental models as the fast path for the project/profile recall slices (single GET, server-refreshed), falling back to recall when absent/empty. (default: true) |
125+
| `memory.external.recall.profile_mental_models` | string[] | `["user-preferences"]` | Main-bank mental-model names (case-insensitive) used for the profile slice. The main bank is never modified by the plugin — create these manually. |
108126
| `embedding` | object | — | Embedding provider configuration |
109127
| `embedding.provider` | `"local"` \\| `"openai-compatible"` \\| `"off"` | `"local"` | Embedding provider. 'local' uses Xenova/all-MiniLM-L6-v2, 'openai-compatible' requires endpoint and model, 'off' disables embeddings. |
110128
| `embedding.model` | string | — | Embedding model name. Required for openai-compatible, ignored for local. |

‎packages/pi-plugin/src/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import {
3535
summarizeDreamSchedule,
3636
userMemoryCollectionEnabled,
3737
} from "@magic-context/core/features/magic-context/dreamer/task-config";
38+
import { initializeExternalMemory } from "@magic-context/core/features/magic-context/memory/external-memory";
3839
import { resolveProjectIdentity } from "@magic-context/core/features/magic-context/memory/project-identity";
3940
import { scheduleIncrementalIndex } from "@magic-context/core/features/magic-context/message-index-async";
4041
import { detectOverflow } from "@magic-context/core/features/magic-context/overflow-detection";
@@ -54,7 +55,6 @@ import {
5455
getOverflowState,
5556
recordOverflowDetected,
5657
} from "@magic-context/core/features/magic-context/storage-meta-persisted";
57-
import { initializeExternalMemory } from "@magic-context/core/features/magic-context/memory/external-memory";
5858
import { runDeferredV22Backfill } from "@magic-context/core/features/magic-context/v22-deferred-backfill";
5959
import {
6060
deriveHistorianChunkTokens,

‎packages/pi-plugin/src/inject-compartments-pi.ts‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,20 +25,19 @@
2525
* historyRefreshSessions signal.
2626
*/
2727

28+
import {
29+
computeRecallSnapshotHash,
30+
type ExternalRecallSnapshot,
31+
readExternalRecallHash,
32+
readExternalRecallSnapshot,
33+
} from "@magic-context/core/features/magic-context/memory/external-recall-read";
2834
import {
2935
getMaxMemoryIdForProjects,
3036
getMemoriesByProject,
3137
getMemoriesByProjects,
3238
readNewMemoriesForM1Union,
3339
} from "@magic-context/core/features/magic-context/memory/storage-memory";
3440
import type { Memory } from "@magic-context/core/features/magic-context/memory/types";
35-
import {
36-
computeRecallSnapshotHash,
37-
type ExternalRecallSnapshot,
38-
readExternalRecallHash,
39-
readExternalRecallSnapshot,
40-
} from "@magic-context/core/features/magic-context/memory/external-recall-read";
41-
import { renderExternalMemoryBlock } from "@magic-context/core/hooks/magic-context/inject-compartments";
4241
import {
4342
type ContextDatabase,
4443
clearCachedM0M1,

‎packages/pi-plugin/src/pi-historian-runner.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,11 @@ import {
4747
appendCompartments,
4848
getCompartments,
4949
} from "@magic-context/core/features/magic-context/compartment-storage";
50+
import type { PromotedMemoryRef } from "@magic-context/core/features/magic-context/memory";
5051
import {
5152
embedPromotedFacts,
5253
promoteSessionFactsDurable,
5354
} from "@magic-context/core/features/magic-context/memory";
54-
import type { PromotedMemoryRef } from "@magic-context/core/features/magic-context/memory";
5555
import { resolveProjectIdentity } from "@magic-context/core/features/magic-context/memory/project-identity";
5656
import { getMemoriesByProject } from "@magic-context/core/features/magic-context/memory/storage-memory";
5757
import {

‎packages/pi-plugin/src/tools/ctx-memory.ts‎

Lines changed: 140 additions & 138 deletions
Original file line numberDiff line numberDiff line change
@@ -390,103 +390,103 @@ export function createCtxMemoryTool(
390390
}
391391
const sessionId = ctx.sessionManager.getSessionId();
392392

393-
// Helper to build an ExternalMemoryRemoveItem from a memory row.
394-
// Identity derives from the original content hash + project identity,
395-
// so a corrective remove needs the row AS IT STOOD before mutation.
396-
const buildRemoveItem = (
397-
memory: { content: string; category: Memory["category"] },
398-
memProjectIdentity: string,
399-
): ExternalMemoryRemoveItem => ({
400-
content: memory.content,
401-
category: memory.category as MemoryCategory,
402-
scope: "project",
403-
projectIdentity: memProjectIdentity,
404-
...(ctx.cwd ? { projectName: basename(ctx.cwd) } : {}),
405-
});
406-
407-
if (params.action === "write") {
408-
const content = params.content?.trim();
409-
if (!content)
410-
return err("Error: 'content' is required when action is 'write'.");
411-
412-
const rawCategory = params.category;
413-
if (!rawCategory) {
414-
return err("Error: 'category' is required when action is 'write'.");
415-
}
393+
// Helper to build an ExternalMemoryRemoveItem from a memory row.
394+
// Identity derives from the original content hash + project identity,
395+
// so a corrective remove needs the row AS IT STOOD before mutation.
396+
const buildRemoveItem = (
397+
memory: { content: string; category: Memory["category"] },
398+
memProjectIdentity: string,
399+
): ExternalMemoryRemoveItem => ({
400+
content: memory.content,
401+
category: memory.category as MemoryCategory,
402+
scope: "project",
403+
projectIdentity: memProjectIdentity,
404+
...(ctx.cwd ? { projectName: basename(ctx.cwd) } : {}),
405+
});
416406

417-
// Global scope: cross-project knowledge goes to the external
418-
// long-term store's main bank ONLY — no local row. Mirrors
419-
// OpenCode's ctx-memory/tools.ts global scope branch.
420-
if (params.scope === "global") {
421-
if (!getExternalMemoryStatus()) {
422-
return err(
423-
"Error: scope 'global' requires an external memory backend (memory.external) — none is configured. Use the default project scope instead.",
407+
if (params.action === "write") {
408+
const content = params.content?.trim();
409+
if (!content)
410+
return err("Error: 'content' is required when action is 'write'.");
411+
412+
const rawCategory = params.category;
413+
if (!rawCategory) {
414+
return err("Error: 'category' is required when action is 'write'.");
415+
}
416+
417+
// Global scope: cross-project knowledge goes to the external
418+
// long-term store's main bank ONLY — no local row. Mirrors
419+
// OpenCode's ctx-memory/tools.ts global scope branch.
420+
if (params.scope === "global") {
421+
if (!getExternalMemoryStatus()) {
422+
return err(
423+
"Error: scope 'global' requires an external memory backend (memory.external) — none is configured. Use the default project scope instead.",
424+
);
425+
}
426+
void teeToExternalBackend("agent", [
427+
{
428+
content,
429+
category: rawCategory,
430+
scope: "global",
431+
projectIdentity,
432+
...(ctx.cwd ? { projectName: basename(ctx.cwd) } : {}),
433+
sourceType: dreamerAllowed ? "dreamer" : "agent",
434+
sessionId,
435+
},
436+
]);
437+
return ok(
438+
`Queued global memory in ${rawCategory} for the long-term store (origin: this project). It has no local ID; it surfaces via the session-start global recall slice and ctx_search source "external" from the next session onward.`,
439+
);
440+
}
441+
442+
const existing = getMemoryByHash(
443+
deps.db,
444+
projectIdentity,
445+
rawCategory,
446+
computeNormalizedHash(content),
447+
);
448+
if (existing) {
449+
updateMemorySeenCount(deps.db, existing.id);
450+
return ok(
451+
`Memory already exists [ID: ${existing.id}] in ${rawCategory} (seen count incremented).`,
424452
);
425453
}
454+
455+
const memory = insertMemory(deps.db, {
456+
projectPath: projectIdentity,
457+
category: rawCategory,
458+
content,
459+
sourceSessionId: sessionId,
460+
sourceType: dreamerAllowed ? "dreamer" : "agent",
461+
});
462+
463+
queueEmbedding({ deps, projectIdentity, memoryId: memory.id, content });
464+
// Do NOT invalidate the m[0]/m[1] cache here. An additive write is a
465+
// supersede-delta operation: it surfaces in m[1] via the maxMemoryId
466+
// watermark (renderM1 reads memories with id > cachedM0MaxMemoryId) on
467+
// the next cache-busting pass, WITHOUT busting m[0]. Clearing the cache
468+
// re-materialized m[0] for every session (the call was global over
469+
// session_meta), defeating the whole additive/non-additive split and
470+
// busting unrelated projects. Matches OpenCode's write path, which
471+
// likewise does no cache invalidation.
472+
473+
// Tee to external backend (project scope). Fire-and-forget; never
474+
// blocks the local write. Mirrors OpenCode's write tee.
426475
void teeToExternalBackend("agent", [
427476
{
428477
content,
429478
category: rawCategory,
430-
scope: "global",
479+
scope: "project",
431480
projectIdentity,
432481
...(ctx.cwd ? { projectName: basename(ctx.cwd) } : {}),
433482
sourceType: dreamerAllowed ? "dreamer" : "agent",
434483
sessionId,
435484
},
436485
]);
437-
return ok(
438-
`Queued global memory in ${rawCategory} for the long-term store (origin: this project). It has no local ID; it surfaces via the session-start global recall slice and ctx_search source "external" from the next session onward.`,
439-
);
440-
}
441486

442-
const existing = getMemoryByHash(
443-
deps.db,
444-
projectIdentity,
445-
rawCategory,
446-
computeNormalizedHash(content),
447-
);
448-
if (existing) {
449-
updateMemorySeenCount(deps.db, existing.id);
450-
return ok(
451-
`Memory already exists [ID: ${existing.id}] in ${rawCategory} (seen count incremented).`,
452-
);
487+
return ok(`Saved memory [ID: ${memory.id}] in ${rawCategory}.`);
453488
}
454489

455-
const memory = insertMemory(deps.db, {
456-
projectPath: projectIdentity,
457-
category: rawCategory,
458-
content,
459-
sourceSessionId: sessionId,
460-
sourceType: dreamerAllowed ? "dreamer" : "agent",
461-
});
462-
463-
queueEmbedding({ deps, projectIdentity, memoryId: memory.id, content });
464-
// Do NOT invalidate the m[0]/m[1] cache here. An additive write is a
465-
// supersede-delta operation: it surfaces in m[1] via the maxMemoryId
466-
// watermark (renderM1 reads memories with id > cachedM0MaxMemoryId) on
467-
// the next cache-busting pass, WITHOUT busting m[0]. Clearing the cache
468-
// re-materialized m[0] for every session (the call was global over
469-
// session_meta), defeating the whole additive/non-additive split and
470-
// busting unrelated projects. Matches OpenCode's write path, which
471-
// likewise does no cache invalidation.
472-
473-
// Tee to external backend (project scope). Fire-and-forget; never
474-
// blocks the local write. Mirrors OpenCode's write tee.
475-
void teeToExternalBackend("agent", [
476-
{
477-
content,
478-
category: rawCategory,
479-
scope: "project",
480-
projectIdentity,
481-
...(ctx.cwd ? { projectName: basename(ctx.cwd) } : {}),
482-
sourceType: dreamerAllowed ? "dreamer" : "agent",
483-
sessionId,
484-
},
485-
]);
486-
487-
return ok(`Saved memory [ID: ${memory.id}] in ${rawCategory}.`);
488-
}
489-
490490
if (params.action === "list") {
491491
const limit = normalizeLimit(params.limit);
492492
const filtered = getMemoriesByProject(deps.db, projectIdentity);
@@ -553,25 +553,27 @@ export function createCtxMemoryTool(
553553
memoryId: memory.id,
554554
content,
555555
});
556-
// W2 corrective propagation: drop the STALE external document (old
557-
// content hash) and tee the corrected fact as a new document. The
558-
// local row's content rewrite already happened in the transaction
559-
// above, so `memory.content` is still the OLD content and `content`
560-
// is the NEW content — both needed for the remove-then-tee cascade.
561-
// Mirrors OpenCode's update W2 path.
562-
void removeFromExternalBackend([buildRemoveItem(memory, targetIdentity)]);
563-
void teeToExternalBackend("agent", [
564-
{
565-
content,
566-
category: memory.category as MemoryCategory,
567-
scope: "project",
568-
projectIdentity: targetIdentity,
569-
...(ctx.cwd ? { projectName: basename(ctx.cwd) } : {}),
570-
sourceType: dreamerAllowed ? "dreamer" : "agent",
571-
sessionId,
572-
},
573-
]);
574-
return ok(`Updated memory [ID: ${memory.id}] in ${memory.category}.`);
556+
// W2 corrective propagation: drop the STALE external document (old
557+
// content hash) and tee the corrected fact as a new document. The
558+
// local row's content rewrite already happened in the transaction
559+
// above, so `memory.content` is still the OLD content and `content`
560+
// is the NEW content — both needed for the remove-then-tee cascade.
561+
// Mirrors OpenCode's update W2 path.
562+
void removeFromExternalBackend([
563+
buildRemoveItem(memory, targetIdentity),
564+
]);
565+
void teeToExternalBackend("agent", [
566+
{
567+
content,
568+
category: memory.category as MemoryCategory,
569+
scope: "project",
570+
projectIdentity: targetIdentity,
571+
...(ctx.cwd ? { projectName: basename(ctx.cwd) } : {}),
572+
sourceType: dreamerAllowed ? "dreamer" : "agent",
573+
sessionId,
574+
},
575+
]);
576+
return ok(`Updated memory [ID: ${memory.id}] in ${memory.category}.`);
575577
}
576578

577579
if (params.action === "merge") {
@@ -860,47 +862,47 @@ export function createCtxMemoryTool(
860862
return ok(`Archived ${plural} [ID: ${idList}]${reasonSuffix}.`);
861863
}
862864

863-
if (params.action === "verify") {
864-
// Dreamer-only: mark a memory as verified and upsert to external
865-
// backend (refreshes Hindsight's recency with zero duplicate risk).
866-
// Mirrors OpenCode's verify action.
867-
const verifyIds = params.ids;
868-
if (
869-
!verifyIds ||
870-
verifyIds.length !== 1 ||
871-
!verifyIds.every(Number.isInteger)
872-
) {
873-
return err(
874-
"Error: 'ids' must contain exactly one integer memory ID when action is 'verify'.",
865+
if (params.action === "verify") {
866+
// Dreamer-only: mark a memory as verified and upsert to external
867+
// backend (refreshes Hindsight's recency with zero duplicate risk).
868+
// Mirrors OpenCode's verify action.
869+
const verifyIds = params.ids;
870+
if (
871+
!verifyIds ||
872+
verifyIds.length !== 1 ||
873+
!verifyIds.every(Number.isInteger)
874+
) {
875+
return err(
876+
"Error: 'ids' must contain exactly one integer memory ID when action is 'verify'.",
877+
);
878+
}
879+
const verifyId = verifyIds[0];
880+
const memory = getMemoryById(deps.db, verifyId);
881+
if (!memory || !memoryVisibleToTool(memory)) {
882+
return err(`Error: Memory with ID ${verifyId} was not found.`);
883+
}
884+
const verifyProjectIdentity = targetIdentityForStoredPath(
885+
memory.projectPath,
875886
);
887+
updateMemoryVerification(deps.db, memory.id, "verified");
888+
// Verbatim re-retain = same document_id = server-side upsert →
889+
// refreshes Hindsight's mentioned_at recency with ZERO duplicate risk.
890+
void upsertToExternalBackend([
891+
{
892+
content: memory.content,
893+
category: memory.category as MemoryCategory,
894+
scope: "project",
895+
projectIdentity: verifyProjectIdentity,
896+
...(ctx.cwd ? { projectName: basename(ctx.cwd) } : {}),
897+
sourceType: "dreamer",
898+
sessionId,
899+
verifiedAt: Date.now(),
900+
},
901+
]);
902+
return ok(`Verified memory [ID: ${memory.id}].`);
876903
}
877-
const verifyId = verifyIds[0];
878-
const memory = getMemoryById(deps.db, verifyId);
879-
if (!memory || !memoryVisibleToTool(memory)) {
880-
return err(`Error: Memory with ID ${verifyId} was not found.`);
881-
}
882-
const verifyProjectIdentity = targetIdentityForStoredPath(
883-
memory.projectPath,
884-
);
885-
updateMemoryVerification(deps.db, memory.id, "verified");
886-
// Verbatim re-retain = same document_id = server-side upsert →
887-
// refreshes Hindsight's mentioned_at recency with ZERO duplicate risk.
888-
void upsertToExternalBackend([
889-
{
890-
content: memory.content,
891-
category: memory.category as MemoryCategory,
892-
scope: "project",
893-
projectIdentity: verifyProjectIdentity,
894-
...(ctx.cwd ? { projectName: basename(ctx.cwd) } : {}),
895-
sourceType: "dreamer",
896-
sessionId,
897-
verifiedAt: Date.now(),
898-
},
899-
]);
900-
return ok(`Verified memory [ID: ${memory.id}].`);
901-
}
902904

903-
return err("Error: Unknown action.");
905+
return err("Error: Unknown action.");
904906
},
905907
};
906908
}

0 commit comments

Comments
 (0)