Model files are downloaded multiple times if progress_callback is active

[thomas-0816]

System Info

e.g.

import { pipeline, env, TextStreamer } from "/node_modules/@huggingface/transformers/dist/transformers.js";
env.localModelPath = '/models/'
env.backends.onnx.wasm.wasmPaths = '/node_modules/onnxruntime-web/dist/';
env.allowRemoteModels = false;
env.allowLocalModels = true;
env.useBrowserCache = false;
env.useWasmCache = false;

const model = await pipeline("text-generation", "Qwen3-4B-ONNX", { dtype: "q4f16", device: "webgpu", progress_callback: () => {} });

with progress_callback:

without progress_callback:

nginx log:

::1 - - [24/Apr/2026:22:25:59 +0200] "GET /transformers.html HTTP/1.1" 304 0 "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:25:59 +0200] "GET /worker.js HTTP/1.1" 200 4580 "http://localhost:8081/transformers.html" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:25:59 +0200] "GET /node_modules/@huggingface/transformers/dist/transformers.js HTTP/1.1" 200 1285352 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:25:59 +0200] "GET /models/Qwen3-4B-ONNX/config.json HTTP/1.1" 200 1780 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:25:59 +0200] "GET /models/Qwen3-4B-ONNX/tokenizer_config.json HTTP/1.1" 200 9761 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:25:59 +0200] "GET /models/Qwen3-4B-ONNX/config.json HTTP/1.1" 200 1780 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:25:59 +0200] "GET /models/Qwen3-4B-ONNX/generation_config.json HTTP/1.1" 200 219 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:25:59 +0200] "GET /models/Qwen3-4B-ONNX/tokenizer.json HTTP/1.1" 200 9117040 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:25:59 +0200] "GET /models/Qwen3-4B-ONNX/tokenizer.json HTTP/1.1" 200 9117040 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:25:59 +0200] "GET /models/Qwen3-4B-ONNX/config.json HTTP/1.1" 200 1780 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:25:59 +0200] "GET /models/Qwen3-4B-ONNX/tokenizer_config.json HTTP/1.1" 200 9761 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:25:59 +0200] "GET /models/Qwen3-4B-ONNX/onnx/model_q4f16.onnx_data_1 HTTP/1.1" 200 62204143 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:25:59 +0200] "GET /models/Qwen3-4B-ONNX/onnx/model_q4f16.onnx_data HTTP/1.1" 200 62976656 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:26:00 +0200] "GET /models/Qwen3-4B-ONNX/onnx/model_q4f16.onnx HTTP/1.1" 200 59762833 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:26:00 +0200] "GET /models/Qwen3-4B-ONNX/generation_config.json HTTP/1.1" 200 219 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:26:03 +0200] "GET /models/Qwen3-4B-ONNX/onnx/model_q4f16.onnx_data_1 HTTP/1.1" 200 677150720 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:26:05 +0200] "GET /models/Qwen3-4B-ONNX/onnx/model_q4f16.onnx_data HTTP/1.1" 200 2096005120 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:26:05 +0200] "GET /node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.asyncify.mjs HTTP/1.1" 200 47389 "http://localhost:8081/node_modules/@huggingface/transformers/dist/transformers.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:26:05 +0200] "GET /node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.asyncify.wasm HTTP/1.1" 200 23567050 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
::1 - - [24/Apr/2026:22:26:05 +0200] "GET /models/Qwen3-4B-ONNX/onnx/model_q4f16.onnx HTTP/1.1" 200 59762833 "http://localhost:8081/worker.js" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"

esp.

::1 - - [24/Apr/2026:22:25:59 +0200] "GET /models/Qwen3-4B-ONNX/onnx/model_q4f16.onnx_data_1 HTTP/1.1" 200 62204143
::1 - - [24/Apr/2026:22:25:59 +0200] "GET /models/Qwen3-4B-ONNX/onnx/model_q4f16.onnx_data HTTP/1.1" 200 62976656
::1 - - [24/Apr/2026:22:26:00 +0200] "GET /models/Qwen3-4B-ONNX/onnx/model_q4f16.onnx HTTP/1.1" 200 59762833
::1 - - [24/Apr/2026:22:26:03 +0200] "GET /models/Qwen3-4B-ONNX/onnx/model_q4f16.onnx_data_1 HTTP/1.1" 200 677150720
::1 - - [24/Apr/2026:22:26:05 +0200] "GET /models/Qwen3-4B-ONNX/onnx/model_q4f16.onnx_data HTTP/1.1" 200 2096005120

Chromium 147.0.7727.101

"@huggingface/transformers": "^4.2.0",

Environment/Platform

• Website/web-app
• Browser extension
• Server-side (e.g., Node.js, Deno, Bun)
• Desktop app (e.g., Electron)
• Other (e.g., VSCode extension)

Description

Duplicate file downloads

Reproduction

Add progress_callback, check web server logs for duplicates

[anishesg]

Took a look, #1664 should address it.