From aa529193c2852ef311b9d685c21064a655b132d2 Mon Sep 17 00:00:00 2001 From: Assaf Vayner Date: Tue, 30 Jun 2026 18:03:32 -0700 Subject: [PATCH 1/2] [hub] Add download command to the CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a `download` command group with three subcommands, one per download library function: - `download file [local-path]` — downloadFile to an arbitrary local path, with a progress bar (xet: true) - `download cache ` — downloadFileToCacheDir into the HF cache dir - `download snapshot ` — snapshotDownload the whole repo into the cache dir Extract a tested `streamBlobToFile` helper to stream a lazy Blob to disk with progress, removing the partial file on error. --- packages/hub/README.md | 16 +- packages/hub/cli.ts | 272 ++++++++++++++++++ .../hub/src/utils/streamBlobToFile.spec.ts | 61 ++++ packages/hub/src/utils/streamBlobToFile.ts | 34 +++ 4 files changed, 382 insertions(+), 1 deletion(-) create mode 100644 packages/hub/src/utils/streamBlobToFile.spec.ts create mode 100644 packages/hub/src/utils/streamBlobToFile.ts diff --git a/packages/hub/README.md b/packages/hub/README.md index ddfee3e338..8f2271deaf 100644 --- a/packages/hub/README.md +++ b/packages/hub/README.md @@ -114,7 +114,7 @@ await hub.deleteRepo({ repo, accessToken: "hf_..." }); ## CLI usage -You can use `@huggingface/hub` in CLI mode to upload files and folders to your repo. +You can use `@huggingface/hub` in CLI mode to upload and download files and folders to/from your repo. ```console npx @huggingface/hub upload coyotte508/test-model . @@ -125,8 +125,18 @@ npx @huggingface/hub upload --repo-type dataset coyotte508/test-dataset . npx @huggingface/hub branch create coyotte508/test-model release --empty npx @huggingface/hub upload coyotte508/test-model . --revision release +# Download a single file to a local path (defaults to ./config.json) +npx @huggingface/hub download file coyotte508/test-model config.json +npx @huggingface/hub download file coyotte508/test-model config.json ./local-config.json +# Download a single file into the Hugging Face cache directory +npx @huggingface/hub download cache coyotte508/test-model config.json +# Download an entire repo into the Hugging Face cache directory +npx @huggingface/hub download snapshot coyotte508/test-model +npx @huggingface/hub download snapshot --repo-type dataset coyotte508/test-dataset + npx @huggingface/hub --help npx @huggingface/hub upload --help +npx @huggingface/hub download --help ``` You can also install globally with `npm install -g @huggingface/hub`. Then you can do: @@ -137,10 +147,14 @@ hfjs upload coyotte508/test-model . hfjs branch create --repo-type dataset coyotte508/test-dataset release --empty hfjs upload --repo-type dataset coyotte508/test-dataset . --revision release +hfjs download file coyotte508/test-model config.json +hfjs download snapshot coyotte508/test-model + hfjs --help hfjs upload --help hfjs help jobs +hfjs help download ``` ## OAuth Login diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 80cf7058c4..6ba7fdd7c4 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -7,17 +7,21 @@ import { createRepo, deleteBranch, deleteRepo, + downloadFile, + downloadFileToCacheDir, getJob, listJobHardware, listJobs, listModels, repoExists, runJob, + snapshotDownload, streamJobLogs, uploadFilesWithProgress, whoAmI, type SpaceHardwareFlavor, } from "./src"; +import { streamBlobToFile } from "./src/utils/streamBlobToFile"; import { pathToFileURL } from "node:url"; import { stat } from "node:fs/promises"; import { basename, join } from "node:path"; @@ -241,6 +245,141 @@ const commands = { }, ] as const, } satisfies SingleCommand, + download: { + description: "Download files or repositories from the Hub", + subcommands: { + file: { + description: "Download a single file to a local path", + args: [ + { + name: "repo-name" as const, + description: + "The name of the repo to download from. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + positional: true, + required: true, + }, + { + name: "path-in-repo" as const, + description: "The path of the file inside the repo to download", + positional: true, + required: true, + }, + { + name: "local-path" as const, + description: "The local path to save the file to. Defaults to the file's basename in the current directory", + positional: true, + }, + { + name: "repo-type" as const, + enum: ["dataset", "model", "space"], + description: + "The type of repo to download from. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + }, + { + name: "revision" as const, + description: "The revision to download from. Defaults to the main branch", + default: "main", + }, + { + name: "quiet" as const, + short: "q", + description: "Suppress all output", + boolean: true, + }, + { + name: "token" as const, + description: + "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN, + }, + ] as const, + }, + cache: { + description: "Download a single file into the local Hugging Face cache directory", + args: [ + { + name: "repo-name" as const, + description: + "The name of the repo to download from. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + positional: true, + required: true, + }, + { + name: "path-in-repo" as const, + description: "The path of the file inside the repo to download", + positional: true, + required: true, + }, + { + name: "cache-dir" as const, + description: "The cache directory to download into. Defaults to the Hugging Face cache directory", + }, + { + name: "repo-type" as const, + enum: ["dataset", "model", "space"], + description: + "The type of repo to download from. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + }, + { + name: "revision" as const, + description: "The revision to download from. Defaults to the main branch", + default: "main", + }, + { + name: "quiet" as const, + short: "q", + description: "Suppress all output", + boolean: true, + }, + { + name: "token" as const, + description: + "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN, + }, + ] as const, + }, + snapshot: { + description: "Download an entire repository into the local Hugging Face cache directory", + args: [ + { + name: "repo-name" as const, + description: + "The name of the repo to download. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + positional: true, + required: true, + }, + { + name: "cache-dir" as const, + description: "The cache directory to download into. Defaults to the Hugging Face cache directory", + }, + { + name: "repo-type" as const, + enum: ["dataset", "model", "space"], + description: + "The type of repo to download. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + }, + { + name: "revision" as const, + description: "The revision to download. Defaults to the main branch", + default: "main", + }, + { + name: "quiet" as const, + short: "q", + description: "Suppress all output", + boolean: true, + }, + { + name: "token" as const, + description: + "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN, + }, + ] as const, + }, + }, + } satisfies CommandGroup, branch: { description: "Manage repository branches", subcommands: { @@ -671,6 +810,139 @@ async function run() { } break; } + case "download": { + const downloadCommandGroup = commands.download; + const currentSubCommandName = subCommandName as keyof typeof downloadCommandGroup.subcommands | undefined; + + // Check if --help is in subcommand position (e.g., "hfjs download --help") + if (subCommandName === "--help" || subCommandName === "-h") { + console.log(listSubcommands("download", downloadCommandGroup)); + break; + } + + // Check if --help is in args position (e.g., "hfjs download file --help") + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + if (currentSubCommandName && downloadCommandGroup.subcommands[currentSubCommandName]) { + console.log(detailedUsageForSubcommand("download", currentSubCommandName)); + } else { + console.log(listSubcommands("download", downloadCommandGroup)); + } + break; + } + + if (!currentSubCommandName || !downloadCommandGroup.subcommands[currentSubCommandName]) { + console.error(`Error: Missing or invalid subcommand for 'download'.`); + console.log(listSubcommands("download", downloadCommandGroup)); + process.exitCode = 1; + break; + } + + const subCmdDef = downloadCommandGroup.subcommands[currentSubCommandName]; + const hubUrl = process.env.HF_ENDPOINT ?? HUB_URL; + + switch (currentSubCommandName) { + case "file": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "download file"); + const { repoName, pathInRepo, localPath, repoType, revision, token, quiet } = parsedArgs; + + const repo = repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName; + + const blob = await downloadFile({ + repo, + path: pathInRepo, + revision, + accessToken: token, + hubUrl, + xet: true, + }); + + if (!blob) { + console.error(`Error: File '${pathInRepo}' not found in repo '${repoName}'.`); + process.exitCode = 1; + break; + } + + const destination = localPath ?? basename(pathInRepo); + + const cliProgress = quiet ? null : await import("cli-progress").catch(() => null); + const bar = + cliProgress && blob.size + ? new cliProgress.SingleBar( + { + clearOnComplete: false, + hideCursor: true, + format: " {bar} | {filename} | {percentage}%", + barCompleteChar: "\u2588", + barIncompleteChar: "\u2591", + }, + cliProgress.Presets.shades_grey, + ) + : null; + bar?.start(blob.size, 0, { filename: basename(destination) }); + + try { + await streamBlobToFile(blob, destination, (bytesWritten) => bar?.update(bytesWritten)); + } finally { + bar?.stop(); + } + + if (!quiet) { + console.log(`\u2705 Downloaded to ${destination}`); + } + break; + } + case "cache": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "download cache"); + const { repoName, pathInRepo, cacheDir, repoType, revision, token, quiet } = parsedArgs; + + const repo = repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName; + + const cachedPath = await downloadFileToCacheDir({ + repo, + path: pathInRepo, + revision, + cacheDir, + accessToken: token, + hubUrl, + }); + + if (!quiet) { + console.log(`\u2705 Cached at ${cachedPath}`); + } + break; + } + case "snapshot": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "download snapshot"); + const { repoName, cacheDir, repoType, revision, token, quiet } = parsedArgs; + + const repo = repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName; + + if (!quiet) { + console.log(`\u2b07\ufe0f Downloading repo '${repoName}'...`); + } + + const snapshotPath = await snapshotDownload({ + repo, + revision, + cacheDir, + accessToken: token, + hubUrl, + }); + + if (!quiet) { + console.log(`\u2705 Downloaded repo to ${snapshotPath}`); + } + break; + } + default: + // Should be caught by the check above + console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'download'.`); + console.log(listSubcommands("download", downloadCommandGroup)); + process.exitCode = 1; + break; + } + break; + } case "branch": { const branchCommandGroup = commands.branch; const currentSubCommandName = subCommandName as keyof typeof branchCommandGroup.subcommands | undefined; diff --git a/packages/hub/src/utils/streamBlobToFile.spec.ts b/packages/hub/src/utils/streamBlobToFile.spec.ts new file mode 100644 index 0000000000..d7475f1249 --- /dev/null +++ b/packages/hub/src/utils/streamBlobToFile.spec.ts @@ -0,0 +1,61 @@ +import { access, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { streamBlobToFile } from "./streamBlobToFile"; + +describe("streamBlobToFile", () => { + let dir: string; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "stream-blob-")); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it("writes the exact bytes of the blob to disk", async () => { + const bytes = new Uint8Array([0, 1, 2, 3, 255, 128, 64]); + const blob = new Blob([bytes]); + const dest = join(dir, "out.bin"); + + await streamBlobToFile(blob, dest); + + const written = await readFile(dest); + expect(new Uint8Array(written)).toEqual(bytes); + }); + + it("reports cumulative progress ending at the blob size", async () => { + const bytes = new Uint8Array(10_000).map((_, i) => i % 256); + const blob = new Blob([bytes]); + const dest = join(dir, "out.bin"); + + const progress: number[] = []; + await streamBlobToFile(blob, dest, (bytesWritten) => progress.push(bytesWritten)); + + expect(progress.length).toBeGreaterThan(0); + // Monotonically increasing + for (let i = 1; i < progress.length; i++) { + expect(progress[i]).toBeGreaterThan(progress[i - 1]); + } + expect(progress[progress.length - 1]).toBe(blob.size); + }); + + it("removes the partial file and rethrows when the stream errors", async () => { + const dest = join(dir, "out.bin"); + const failing = { + size: 100, + stream: () => + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1, 2, 3])); + controller.error(new Error("boom")); + }, + }), + } as unknown as Blob; + + await expect(streamBlobToFile(failing, dest)).rejects.toThrow("boom"); + await expect(access(dest)).rejects.toThrow(); + }); +}); diff --git a/packages/hub/src/utils/streamBlobToFile.ts b/packages/hub/src/utils/streamBlobToFile.ts new file mode 100644 index 0000000000..4c533028fe --- /dev/null +++ b/packages/hub/src/utils/streamBlobToFile.ts @@ -0,0 +1,34 @@ +import { createWriteStream } from "node:fs"; +import { unlink } from "node:fs/promises"; +import { Readable } from "node:stream"; +import type { ReadableStream } from "node:stream/web"; +import { pipeline } from "node:stream/promises"; + +/** + * Stream a (potentially lazy) Blob to a local file path. + * + * @param onProgress called after each chunk with the cumulative number of bytes written so far. + * On error, the partially written file is removed so no truncated file is left behind. + */ +export async function streamBlobToFile( + blob: Blob, + filePath: string, + onProgress?: (bytesWritten: number) => void, +): Promise { + let bytesWritten = 0; + const source = Readable.fromWeb(blob.stream() as ReadableStream); + + if (onProgress) { + source.on("data", (chunk: Buffer) => { + bytesWritten += chunk.byteLength; + onProgress(bytesWritten); + }); + } + + try { + await pipeline(source, createWriteStream(filePath)); + } catch (error) { + await unlink(filePath).catch(() => {}); + throw error; + } +} From f9e1d4b3f60d4f3a2bbf29741406ca703d41dbaa Mon Sep 17 00:00:00 2001 From: Assaf Vayner Date: Wed, 1 Jul 2026 13:32:41 -0700 Subject: [PATCH 2/2] Move streamBlobToFile out of utils into the CLI --- packages/hub/cli.ts | 36 ++++++++++- .../hub/src/utils/streamBlobToFile.spec.ts | 61 ------------------- packages/hub/src/utils/streamBlobToFile.ts | 34 ----------- 3 files changed, 34 insertions(+), 97 deletions(-) delete mode 100644 packages/hub/src/utils/streamBlobToFile.spec.ts delete mode 100644 packages/hub/src/utils/streamBlobToFile.ts diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 6ba7fdd7c4..77e616cab6 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -21,15 +21,47 @@ import { whoAmI, type SpaceHardwareFlavor, } from "./src"; -import { streamBlobToFile } from "./src/utils/streamBlobToFile"; import { pathToFileURL } from "node:url"; -import { stat } from "node:fs/promises"; +import { createWriteStream } from "node:fs"; +import { stat, unlink } from "node:fs/promises"; +import { Readable } from "node:stream"; +import type { ReadableStream } from "node:stream/web"; +import { pipeline } from "node:stream/promises"; import { basename, join } from "node:path"; import { HUB_URL } from "./src/consts"; import { version } from "./package.json"; import type { CommitProgressEvent } from "./src/lib/commit"; import type { MultiBar, SingleBar } from "cli-progress"; +/** + * Stream a (potentially lazy) Blob to a local file path. + * + * @param onProgress called after each chunk with the cumulative number of bytes written so far. + * On error, the partially written file is removed so no truncated file is left behind. + */ +async function streamBlobToFile( + blob: Blob, + filePath: string, + onProgress?: (bytesWritten: number) => void, +): Promise { + let bytesWritten = 0; + const source = Readable.fromWeb(blob.stream() as ReadableStream); + + if (onProgress) { + source.on("data", (chunk: Buffer) => { + bytesWritten += chunk.byteLength; + onProgress(bytesWritten); + }); + } + + try { + await pipeline(source, createWriteStream(filePath)); + } catch (error) { + await unlink(filePath).catch(() => {}); + throw error; + } +} + // Progress bar manager for handling multiple file uploads class UploadProgressManager { private multibar: MultiBar | null = null; diff --git a/packages/hub/src/utils/streamBlobToFile.spec.ts b/packages/hub/src/utils/streamBlobToFile.spec.ts deleted file mode 100644 index d7475f1249..0000000000 --- a/packages/hub/src/utils/streamBlobToFile.spec.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { access, mkdtemp, readFile, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { streamBlobToFile } from "./streamBlobToFile"; - -describe("streamBlobToFile", () => { - let dir: string; - - beforeEach(async () => { - dir = await mkdtemp(join(tmpdir(), "stream-blob-")); - }); - - afterEach(async () => { - await rm(dir, { recursive: true, force: true }); - }); - - it("writes the exact bytes of the blob to disk", async () => { - const bytes = new Uint8Array([0, 1, 2, 3, 255, 128, 64]); - const blob = new Blob([bytes]); - const dest = join(dir, "out.bin"); - - await streamBlobToFile(blob, dest); - - const written = await readFile(dest); - expect(new Uint8Array(written)).toEqual(bytes); - }); - - it("reports cumulative progress ending at the blob size", async () => { - const bytes = new Uint8Array(10_000).map((_, i) => i % 256); - const blob = new Blob([bytes]); - const dest = join(dir, "out.bin"); - - const progress: number[] = []; - await streamBlobToFile(blob, dest, (bytesWritten) => progress.push(bytesWritten)); - - expect(progress.length).toBeGreaterThan(0); - // Monotonically increasing - for (let i = 1; i < progress.length; i++) { - expect(progress[i]).toBeGreaterThan(progress[i - 1]); - } - expect(progress[progress.length - 1]).toBe(blob.size); - }); - - it("removes the partial file and rethrows when the stream errors", async () => { - const dest = join(dir, "out.bin"); - const failing = { - size: 100, - stream: () => - new ReadableStream({ - start(controller) { - controller.enqueue(new Uint8Array([1, 2, 3])); - controller.error(new Error("boom")); - }, - }), - } as unknown as Blob; - - await expect(streamBlobToFile(failing, dest)).rejects.toThrow("boom"); - await expect(access(dest)).rejects.toThrow(); - }); -}); diff --git a/packages/hub/src/utils/streamBlobToFile.ts b/packages/hub/src/utils/streamBlobToFile.ts deleted file mode 100644 index 4c533028fe..0000000000 --- a/packages/hub/src/utils/streamBlobToFile.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { createWriteStream } from "node:fs"; -import { unlink } from "node:fs/promises"; -import { Readable } from "node:stream"; -import type { ReadableStream } from "node:stream/web"; -import { pipeline } from "node:stream/promises"; - -/** - * Stream a (potentially lazy) Blob to a local file path. - * - * @param onProgress called after each chunk with the cumulative number of bytes written so far. - * On error, the partially written file is removed so no truncated file is left behind. - */ -export async function streamBlobToFile( - blob: Blob, - filePath: string, - onProgress?: (bytesWritten: number) => void, -): Promise { - let bytesWritten = 0; - const source = Readable.fromWeb(blob.stream() as ReadableStream); - - if (onProgress) { - source.on("data", (chunk: Buffer) => { - bytesWritten += chunk.byteLength; - onProgress(bytesWritten); - }); - } - - try { - await pipeline(source, createWriteStream(filePath)); - } catch (error) { - await unlink(filePath).catch(() => {}); - throw error; - } -}