diff --git a/packages/hub/README.md b/packages/hub/README.md index ddfee3e338..8f2271deaf 100644 --- a/packages/hub/README.md +++ b/packages/hub/README.md @@ -114,7 +114,7 @@ await hub.deleteRepo({ repo, accessToken: "hf_..." }); ## CLI usage -You can use `@huggingface/hub` in CLI mode to upload files and folders to your repo. +You can use `@huggingface/hub` in CLI mode to upload and download files and folders to/from your repo. ```console npx @huggingface/hub upload coyotte508/test-model . @@ -125,8 +125,18 @@ npx @huggingface/hub upload --repo-type dataset coyotte508/test-dataset . npx @huggingface/hub branch create coyotte508/test-model release --empty npx @huggingface/hub upload coyotte508/test-model . --revision release +# Download a single file to a local path (defaults to ./config.json) +npx @huggingface/hub download file coyotte508/test-model config.json +npx @huggingface/hub download file coyotte508/test-model config.json ./local-config.json +# Download a single file into the Hugging Face cache directory +npx @huggingface/hub download cache coyotte508/test-model config.json +# Download an entire repo into the Hugging Face cache directory +npx @huggingface/hub download snapshot coyotte508/test-model +npx @huggingface/hub download snapshot --repo-type dataset coyotte508/test-dataset + npx @huggingface/hub --help npx @huggingface/hub upload --help +npx @huggingface/hub download --help ``` You can also install globally with `npm install -g @huggingface/hub`. Then you can do: @@ -137,10 +147,14 @@ hfjs upload coyotte508/test-model . hfjs branch create --repo-type dataset coyotte508/test-dataset release --empty hfjs upload --repo-type dataset coyotte508/test-dataset . --revision release +hfjs download file coyotte508/test-model config.json +hfjs download snapshot coyotte508/test-model + hfjs --help hfjs upload --help hfjs help jobs +hfjs help download ``` ## OAuth Login diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 80cf7058c4..77e616cab6 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -7,25 +7,61 @@ import { createRepo, deleteBranch, deleteRepo, + downloadFile, + downloadFileToCacheDir, getJob, listJobHardware, listJobs, listModels, repoExists, runJob, + snapshotDownload, streamJobLogs, uploadFilesWithProgress, whoAmI, type SpaceHardwareFlavor, } from "./src"; import { pathToFileURL } from "node:url"; -import { stat } from "node:fs/promises"; +import { createWriteStream } from "node:fs"; +import { stat, unlink } from "node:fs/promises"; +import { Readable } from "node:stream"; +import type { ReadableStream } from "node:stream/web"; +import { pipeline } from "node:stream/promises"; import { basename, join } from "node:path"; import { HUB_URL } from "./src/consts"; import { version } from "./package.json"; import type { CommitProgressEvent } from "./src/lib/commit"; import type { MultiBar, SingleBar } from "cli-progress"; +/** + * Stream a (potentially lazy) Blob to a local file path. + * + * @param onProgress called after each chunk with the cumulative number of bytes written so far. + * On error, the partially written file is removed so no truncated file is left behind. + */ +async function streamBlobToFile( + blob: Blob, + filePath: string, + onProgress?: (bytesWritten: number) => void, +): Promise { + let bytesWritten = 0; + const source = Readable.fromWeb(blob.stream() as ReadableStream); + + if (onProgress) { + source.on("data", (chunk: Buffer) => { + bytesWritten += chunk.byteLength; + onProgress(bytesWritten); + }); + } + + try { + await pipeline(source, createWriteStream(filePath)); + } catch (error) { + await unlink(filePath).catch(() => {}); + throw error; + } +} + // Progress bar manager for handling multiple file uploads class UploadProgressManager { private multibar: MultiBar | null = null; @@ -241,6 +277,141 @@ const commands = { }, ] as const, } satisfies SingleCommand, + download: { + description: "Download files or repositories from the Hub", + subcommands: { + file: { + description: "Download a single file to a local path", + args: [ + { + name: "repo-name" as const, + description: + "The name of the repo to download from. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + positional: true, + required: true, + }, + { + name: "path-in-repo" as const, + description: "The path of the file inside the repo to download", + positional: true, + required: true, + }, + { + name: "local-path" as const, + description: "The local path to save the file to. Defaults to the file's basename in the current directory", + positional: true, + }, + { + name: "repo-type" as const, + enum: ["dataset", "model", "space"], + description: + "The type of repo to download from. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + }, + { + name: "revision" as const, + description: "The revision to download from. Defaults to the main branch", + default: "main", + }, + { + name: "quiet" as const, + short: "q", + description: "Suppress all output", + boolean: true, + }, + { + name: "token" as const, + description: + "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN, + }, + ] as const, + }, + cache: { + description: "Download a single file into the local Hugging Face cache directory", + args: [ + { + name: "repo-name" as const, + description: + "The name of the repo to download from. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + positional: true, + required: true, + }, + { + name: "path-in-repo" as const, + description: "The path of the file inside the repo to download", + positional: true, + required: true, + }, + { + name: "cache-dir" as const, + description: "The cache directory to download into. Defaults to the Hugging Face cache directory", + }, + { + name: "repo-type" as const, + enum: ["dataset", "model", "space"], + description: + "The type of repo to download from. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + }, + { + name: "revision" as const, + description: "The revision to download from. Defaults to the main branch", + default: "main", + }, + { + name: "quiet" as const, + short: "q", + description: "Suppress all output", + boolean: true, + }, + { + name: "token" as const, + description: + "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN, + }, + ] as const, + }, + snapshot: { + description: "Download an entire repository into the local Hugging Face cache directory", + args: [ + { + name: "repo-name" as const, + description: + "The name of the repo to download. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + positional: true, + required: true, + }, + { + name: "cache-dir" as const, + description: "The cache directory to download into. Defaults to the Hugging Face cache directory", + }, + { + name: "repo-type" as const, + enum: ["dataset", "model", "space"], + description: + "The type of repo to download. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + }, + { + name: "revision" as const, + description: "The revision to download. Defaults to the main branch", + default: "main", + }, + { + name: "quiet" as const, + short: "q", + description: "Suppress all output", + boolean: true, + }, + { + name: "token" as const, + description: + "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN, + }, + ] as const, + }, + }, + } satisfies CommandGroup, branch: { description: "Manage repository branches", subcommands: { @@ -671,6 +842,139 @@ async function run() { } break; } + case "download": { + const downloadCommandGroup = commands.download; + const currentSubCommandName = subCommandName as keyof typeof downloadCommandGroup.subcommands | undefined; + + // Check if --help is in subcommand position (e.g., "hfjs download --help") + if (subCommandName === "--help" || subCommandName === "-h") { + console.log(listSubcommands("download", downloadCommandGroup)); + break; + } + + // Check if --help is in args position (e.g., "hfjs download file --help") + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + if (currentSubCommandName && downloadCommandGroup.subcommands[currentSubCommandName]) { + console.log(detailedUsageForSubcommand("download", currentSubCommandName)); + } else { + console.log(listSubcommands("download", downloadCommandGroup)); + } + break; + } + + if (!currentSubCommandName || !downloadCommandGroup.subcommands[currentSubCommandName]) { + console.error(`Error: Missing or invalid subcommand for 'download'.`); + console.log(listSubcommands("download", downloadCommandGroup)); + process.exitCode = 1; + break; + } + + const subCmdDef = downloadCommandGroup.subcommands[currentSubCommandName]; + const hubUrl = process.env.HF_ENDPOINT ?? HUB_URL; + + switch (currentSubCommandName) { + case "file": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "download file"); + const { repoName, pathInRepo, localPath, repoType, revision, token, quiet } = parsedArgs; + + const repo = repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName; + + const blob = await downloadFile({ + repo, + path: pathInRepo, + revision, + accessToken: token, + hubUrl, + xet: true, + }); + + if (!blob) { + console.error(`Error: File '${pathInRepo}' not found in repo '${repoName}'.`); + process.exitCode = 1; + break; + } + + const destination = localPath ?? basename(pathInRepo); + + const cliProgress = quiet ? null : await import("cli-progress").catch(() => null); + const bar = + cliProgress && blob.size + ? new cliProgress.SingleBar( + { + clearOnComplete: false, + hideCursor: true, + format: " {bar} | {filename} | {percentage}%", + barCompleteChar: "\u2588", + barIncompleteChar: "\u2591", + }, + cliProgress.Presets.shades_grey, + ) + : null; + bar?.start(blob.size, 0, { filename: basename(destination) }); + + try { + await streamBlobToFile(blob, destination, (bytesWritten) => bar?.update(bytesWritten)); + } finally { + bar?.stop(); + } + + if (!quiet) { + console.log(`\u2705 Downloaded to ${destination}`); + } + break; + } + case "cache": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "download cache"); + const { repoName, pathInRepo, cacheDir, repoType, revision, token, quiet } = parsedArgs; + + const repo = repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName; + + const cachedPath = await downloadFileToCacheDir({ + repo, + path: pathInRepo, + revision, + cacheDir, + accessToken: token, + hubUrl, + }); + + if (!quiet) { + console.log(`\u2705 Cached at ${cachedPath}`); + } + break; + } + case "snapshot": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "download snapshot"); + const { repoName, cacheDir, repoType, revision, token, quiet } = parsedArgs; + + const repo = repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName; + + if (!quiet) { + console.log(`\u2b07\ufe0f Downloading repo '${repoName}'...`); + } + + const snapshotPath = await snapshotDownload({ + repo, + revision, + cacheDir, + accessToken: token, + hubUrl, + }); + + if (!quiet) { + console.log(`\u2705 Downloaded repo to ${snapshotPath}`); + } + break; + } + default: + // Should be caught by the check above + console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'download'.`); + console.log(listSubcommands("download", downloadCommandGroup)); + process.exitCode = 1; + break; + } + break; + } case "branch": { const branchCommandGroup = commands.branch; const currentSubCommandName = subCommandName as keyof typeof branchCommandGroup.subcommands | undefined;