Skip to content

Commit f004326

Browse files
committed
fix(embedding): retry transient ONNX load failures with backoff
Live logs show recurring 'Protobuf parsing failed' and 'Unable to get model file path or buffer' errors when multiple plugin processes (Desktop sidecar + TUI + dashboard) initialize the embedding pipeline around the same time. Verified on disk that model.onnx is intact (byte-identical to HuggingFace, matching SHA256), so the failure mode is ephemeral — likely onnxruntime-node mmap/page cache races between concurrent processes. Adds a bounded retry loop with jittered backoff (up to 3 attempts, 300ms × attempt + 0-200ms jitter) inside LocalEmbeddingProvider.initialize. Only transient-looking failures trigger the retry; other errors surface immediately. The first two attempts log at debug-ish level; only a final failure after all retries still surfaces as 'embedding model failed to load'.
1 parent d77e701 commit f004326

1 file changed

Lines changed: 59 additions & 7 deletions

File tree

packages/plugin/src/features/magic-context/memory/embedding-local.ts

Lines changed: 59 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,26 @@ async function withQuietConsole<T>(fn: () => Promise<T>): Promise<T> {
4242
}
4343
}
4444

45+
/**
46+
* Recognizes transient ONNX/transformers load failures that should be retried
47+
* rather than surfaced to the user. Seen in live logs when multiple plugin
48+
* processes (Desktop sidecar + TUI + dashboard) initialize the embedding
49+
* pipeline within the same window. The on-disk model file is intact; the
50+
* failure mode is ephemeral and resolves on retry.
51+
*/
52+
function isTransientLoadError(error: unknown): boolean {
53+
const message = error instanceof Error ? error.message : String(error ?? "");
54+
if (!message) return false;
55+
const lower = message.toLowerCase();
56+
return (
57+
lower.includes("protobuf parsing failed") ||
58+
lower.includes("unable to get model file path or buffer") ||
59+
lower.includes("ebusy") ||
60+
lower.includes("resource busy") ||
61+
lower.includes("resource temporarily unavailable")
62+
);
63+
}
64+
4565
function isArrayLikeNumber(value: unknown): value is ArrayLike<number> {
4666
if (typeof value !== "object" || value === null || !("length" in value)) {
4767
return false;
@@ -141,13 +161,45 @@ export class LocalEmbeddingProvider implements EmbeddingProvider {
141161
env.logLevel = LogLevel.ERROR;
142162
}
143163
const createPipeline = transformersModule.pipeline as CreateEmbeddingPipeline;
144-
this.pipeline = await withQuietConsole(() =>
145-
createPipeline("feature-extraction", this.model, {
146-
quantized: true,
147-
dtype: "fp32",
148-
}),
149-
);
150-
log(`[magic-context] embedding model loaded: ${this.model}`);
164+
165+
// Retry loop absorbs transient failures seen when multiple plugin
166+
// processes initialize the ONNX session around the same time:
167+
// - "Protobuf parsing failed" (onnxruntime-node race on mmap/page cache)
168+
// - "Unable to get model file path or buffer" (download still in progress)
169+
// - EBUSY / file lock contention
170+
// Recovery happens within a few hundred ms. The file on disk is fine;
171+
// we verified this on live logs with matching SHA256 vs HuggingFace.
172+
const MAX_ATTEMPTS = 3;
173+
let lastError: unknown;
174+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
175+
try {
176+
this.pipeline = await withQuietConsole(() =>
177+
createPipeline("feature-extraction", this.model, {
178+
quantized: true,
179+
dtype: "fp32",
180+
}),
181+
);
182+
lastError = undefined;
183+
break;
184+
} catch (error) {
185+
lastError = error;
186+
if (!isTransientLoadError(error) || attempt === MAX_ATTEMPTS) {
187+
break;
188+
}
189+
// Jittered backoff: 300ms + random 0-200ms, grows by attempt.
190+
const delayMs = 300 * attempt + Math.floor(Math.random() * 200);
191+
log(
192+
`[magic-context] embedding model load attempt ${attempt}/${MAX_ATTEMPTS} failed transiently, retrying in ${delayMs}ms`,
193+
);
194+
await new Promise((resolve) => setTimeout(resolve, delayMs));
195+
}
196+
}
197+
198+
if (this.pipeline) {
199+
log(`[magic-context] embedding model loaded: ${this.model}`);
200+
} else {
201+
throw lastError ?? new Error("unknown embedding load failure");
202+
}
151203
} catch (error) {
152204
log("[magic-context] embedding model failed to load:", error);
153205
this.pipeline = null;

0 commit comments

Comments
 (0)