diff --git a/.github/workflows/kernel-builder-cli-docs.yaml b/.github/workflows/kernel-builder-cli-docs.yaml new file mode 100644 index 00000000..606da7e6 --- /dev/null +++ b/.github/workflows/kernel-builder-cli-docs.yaml @@ -0,0 +1,45 @@ +name: Check CLI docs are up to date + +on: + pull_request: + paths: + - "kernel-builder/src/**" + - "kernel-builder/Cargo.toml" + - "docs/source/builder-cli.md" + - "Makefile" + - ".github/workflows/kernel-builder-cli-docs.yaml" + push: + branches: + - main + +jobs: + check-cli-docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + + - name: Cache cargo registry and build + uses: actions/cache@a7833574556fa59680c1b7cb190c1735db73ebf0 # v5.0.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-cli-docs-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-cli-docs- + + - name: Generate CLI docs + run: make kernel-builder-cli-docs + + - name: Check docs are up to date + run: | + if ! git diff --quiet docs/source/builder-cli.md; then + echo "::error::builder-cli.md is out of date. Run 'make kernel-builder-cli-docs' to regenerate." + git diff docs/source/builder-cli.md + exit 1 + fi + echo "CLI docs are up to date." diff --git a/Cargo.lock b/Cargo.lock index cbf833ea..95f32cf3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -313,6 +313,15 @@ dependencies = [ "clap_derive", ] +[[package]] +name = "clap-markdown" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2a2617956a06d4885b490697b5307ebb09fec10b088afc18c81762d848c2339" +dependencies = [ + "clap", +] + [[package]] name = "clap_builder" version = "4.6.0" @@ -1042,6 +1051,7 @@ version = "0.13.0-dev0" dependencies = [ "base32", "clap", + "clap-markdown", "clap_complete", "dirs", "eyre", diff --git a/Makefile b/Makefile index c86d71b1..cb528a7b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: style +.PHONY: style kernel-builder-cli-docs export check_dirs := kernels/src kernels/tests @@ -11,3 +11,12 @@ style: black ${check_dirs} isort ${check_dirs} ruff check ${check_dirs} --fix + +kernel-builder-cli-docs: + cargo build -p hf-kernel-builder + ./target/debug/kernel-builder generate-docs \ + | sed 's/hf-kernel-builder/kernel-builder/g' \ + | sed '1s/^# Command-Line Help for `kernel-builder`/# CLI reference for kernel-builder/' \ + | sed '/`--backends/,/^\*/{/^ Default value:/d;}' \ + > docs/source/builder-cli.md + @echo "Generated docs/source/builder-cli.md" diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 12eac1b9..e56c747f 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -67,4 +67,6 @@ title: kernels download - local: cli-skills title: kernel-builder skills + - local: builder-cli + title: Builder CLI Reference title: CLI Reference diff --git a/docs/source/builder-cli.md b/docs/source/builder-cli.md new file mode 100644 index 00000000..42927e20 --- /dev/null +++ b/docs/source/builder-cli.md @@ -0,0 +1,327 @@ +# CLI reference for kernel-builder + +This document contains the help content for the `kernel-builder` command-line program. + +**Command Overview:** + +* [`kernel-builder`↴](#kernel-builder) +* [`kernel-builder completions`↴](#kernel-builder-completions) +* [`kernel-builder init`↴](#kernel-builder-init) +* [`kernel-builder build`↴](#kernel-builder-build) +* [`kernel-builder build-and-copy`↴](#kernel-builder-build-and-copy) +* [`kernel-builder build-and-upload`↴](#kernel-builder-build-and-upload) +* [`kernel-builder upload`↴](#kernel-builder-upload) +* [`kernel-builder create-pyproject`↴](#kernel-builder-create-pyproject) +* [`kernel-builder devshell`↴](#kernel-builder-devshell) +* [`kernel-builder list-variants`↴](#kernel-builder-list-variants) +* [`kernel-builder testshell`↴](#kernel-builder-testshell) +* [`kernel-builder update-build`↴](#kernel-builder-update-build) +* [`kernel-builder validate`↴](#kernel-builder-validate) +* [`kernel-builder skills`↴](#kernel-builder-skills) +* [`kernel-builder skills add`↴](#kernel-builder-skills-add) +* [`kernel-builder clean-pyproject`↴](#kernel-builder-clean-pyproject) + +## `kernel-builder` + +Build Hugging Face Hub kernels + +**Usage:** `kernel-builder ` + +###### **Subcommands:** + +* `completions` — Generate shell completions +* `init` — Initialize a new kernel project from template +* `build` — Build the kernel locally (alias for build-and-copy) +* `build-and-copy` — Build the kernel and copy artifacts locally +* `build-and-upload` — Build the kernel and upload to Hugging Face Hub +* `upload` — Upload kernel build artifacts to the Hugging Face Hub +* `create-pyproject` — Generate CMake files for a kernel extension build +* `devshell` — Spawn a kernel development shell +* `list-variants` — List build variants +* `testshell` — Spawn a kernel test shell +* `update-build` — Update a `build.toml` to the current format +* `validate` — Validate the build.toml file +* `skills` — Install skills for AI coding assistants (Claude, Codex, OpenCode) +* `clean-pyproject` — Clean generated artifacts + + + +## `kernel-builder completions` + +Generate shell completions + +**Usage:** `kernel-builder completions ` + +###### **Arguments:** + +* `` + + Possible values: `bash`, `elvish`, `fish`, `powershell`, `zsh` + + + + +## `kernel-builder init` + +Initialize a new kernel project from template + +**Usage:** `kernel-builder init [OPTIONS] [PATH]` + +###### **Arguments:** + +* `` — Directory to initialize (defaults to current directory) + +###### **Options:** + +* `--name ` — Name of the kernel repo (e.g. `drbh/my-kernel`) +* `--backends ` — Backends to enable (`all`, `cpu`, `cuda`, `metal`, `neuron`, `rocm`, `xpu`) + +* `--overwrite` — Overwrite existing scaffold files (preserves other files) + + + +## `kernel-builder build` + +Build the kernel locally (alias for build-and-copy) + +**Usage:** `kernel-builder build [OPTIONS] [KERNEL_DIR]` + +###### **Arguments:** + +* `` — Directory of the kernel project (defaults to current directory) + +###### **Options:** + +* `--variant ` — Build a specific variant +* `--max-jobs ` — Maximum number of parallel Nix build jobs +* `--cores ` — Number of CPU cores to use for each build job +* `-L`, `--print-build-logs` — Print full build logs on standard error + + + +## `kernel-builder build-and-copy` + +Build the kernel and copy artifacts locally + +**Usage:** `kernel-builder build-and-copy [OPTIONS] [KERNEL_DIR]` + +###### **Arguments:** + +* `` — Directory of the kernel project (defaults to current directory) + +###### **Options:** + +* `--max-jobs ` — Maximum number of parallel Nix build jobs +* `--cores ` — Number of CPU cores to use for each build job +* `-L`, `--print-build-logs` — Print full build logs on standard error + + + +## `kernel-builder build-and-upload` + +Build the kernel and upload to Hugging Face Hub + +**Usage:** `kernel-builder build-and-upload [OPTIONS] [KERNEL_DIR]` + +###### **Arguments:** + +* `` — Directory of the kernel project (defaults to current directory) + +###### **Options:** + +* `--variant ` — Build a specific variant +* `--max-jobs ` — Maximum number of parallel Nix build jobs +* `--cores ` — Number of CPU cores to use for each build job +* `-L`, `--print-build-logs` — Print full build logs on standard error +* `--repo-id ` — Repository ID on the Hugging Face Hub (e.g. `user/my-kernel`) +* `--branch ` — Upload to a specific branch (defaults to `v{version}` from metadata) +* `--private` — Create the repository as private +* `--repo-type ` — Repository type on Hugging Face Hub (`kernel` by default, or `model` for legacy repos) + + Default value: `kernel` + + Possible values: `model`, `kernel` + + + + +## `kernel-builder upload` + +Upload kernel build artifacts to the Hugging Face Hub + +**Usage:** `kernel-builder upload [OPTIONS] [KERNEL_DIR]` + +###### **Arguments:** + +* `` — Directory of the kernel build (defaults to current directory) + +###### **Options:** + +* `--repo-id ` — Repository ID on the Hugging Face Hub (e.g. `user/my-kernel`). Defaults to `general.hub.repo-id` from `build.toml` +* `--branch ` — Upload to a specific branch (defaults to `v{version}` from metadata) +* `--private` — Create the repository as private +* `--repo-type ` — Repository type on Hugging Face Hub (`kernel` by default, or `model` for legacy repos) + + Default value: `kernel` + + Possible values: `model`, `kernel` + + + + +## `kernel-builder create-pyproject` + +Generate CMake files for a kernel extension build + +**Usage:** `kernel-builder create-pyproject [OPTIONS] [KERNEL_DIR] [TARGET_DIR]` + +###### **Arguments:** + +* `` +* `` — The directory to write the generated files to (directory of `BUILD_TOML` when absent) + +###### **Options:** + +* `-f`, `--force` — Force-overwrite existing files +* `--ops-id ` — This is an optional unique identifier that is suffixed to the kernel name to avoid name collisions. (e.g. Git SHA) + + + +## `kernel-builder devshell` + +Spawn a kernel development shell + +**Usage:** `kernel-builder devshell [OPTIONS] [KERNEL_DIR]` + +###### **Arguments:** + +* `` + +###### **Options:** + +* `--variant ` — Use a specific variant +* `--max-jobs ` — Maximum number of parallel Nix build jobs +* `--cores ` — Number of CPU cores to use for each build job +* `-L`, `--print-build-logs` — Print full build logs on standard error + + + +## `kernel-builder list-variants` + +List build variants + +**Usage:** `kernel-builder list-variants [OPTIONS] [KERNEL_DIR]` + +###### **Arguments:** + +* `` + +###### **Options:** + +* `--arch` — Only list variants for the current architecture + + + +## `kernel-builder testshell` + +Spawn a kernel test shell + +**Usage:** `kernel-builder testshell [OPTIONS] [KERNEL_DIR]` + +###### **Arguments:** + +* `` + +###### **Options:** + +* `--variant ` — Use a specific variant +* `--max-jobs ` — Maximum number of parallel Nix build jobs +* `--cores ` — Number of CPU cores to use for each build job +* `-L`, `--print-build-logs` — Print full build logs on standard error + + + +## `kernel-builder update-build` + +Update a `build.toml` to the current format + +**Usage:** `kernel-builder update-build [KERNEL_DIR]` + +###### **Arguments:** + +* `` + + + +## `kernel-builder validate` + +Validate the build.toml file + +**Usage:** `kernel-builder validate [KERNEL_DIR]` + +###### **Arguments:** + +* `` + + + +## `kernel-builder skills` + +Install skills for AI coding assistants (Claude, Codex, OpenCode) + +**Usage:** `kernel-builder skills ` + +###### **Subcommands:** + +* `add` — Install a kernels skill for an AI assistant + + + +## `kernel-builder skills add` + +Install a kernels skill for an AI assistant + +**Usage:** `kernel-builder skills add [OPTIONS]` + +###### **Options:** + +* `--skill ` — Skill to install + + Default value: `cuda-kernels` + + Possible values: `cuda-kernels`, `rocm-kernels` + +* `--claude` — Install for Claude +* `--codex` — Install for Codex +* `--opencode` — Install for OpenCode +* `-g`, `--global` — Install globally (user-level) instead of in the current project directory +* `--dest ` — Install into a custom destination (path to skills directory) +* `--force` — Overwrite existing skills in the destination + + + +## `kernel-builder clean-pyproject` + +Clean generated artifacts + +**Usage:** `kernel-builder clean-pyproject [OPTIONS] [KERNEL_DIR] [TARGET_DIR]` + +###### **Arguments:** + +* `` +* `` — The directory to clean from (directory of `BUILD_TOML` when absent) + +###### **Options:** + +* `-d`, `--dry-run` — Show what would be deleted without actually deleting +* `-f`, `--force` — Force deletion without confirmation +* `--ops-id ` — This is an optional unique identifier that is suffixed to the kernel name to avoid name collisions. (e.g. Git SHA) + + + +
+ + + This document was generated automatically by + clap-markdown. + diff --git a/kernel-builder/Cargo.toml b/kernel-builder/Cargo.toml index 28165b8c..31f8042b 100644 --- a/kernel-builder/Cargo.toml +++ b/kernel-builder/Cargo.toml @@ -12,6 +12,7 @@ repository = "https://github.com/huggingface/kernels" base32 = "0.5" kernels-data = { path = "../kernels-data" } clap = { version = "4", features = ["derive"] } +clap-markdown = "0.1.5" clap_complete = "4" eyre = "0.6.12" git2 = "0.20" diff --git a/kernel-builder/src/main.rs b/kernel-builder/src/main.rs index 19158f49..aa1dbbbe 100644 --- a/kernel-builder/src/main.rs +++ b/kernel-builder/src/main.rs @@ -199,6 +199,10 @@ enum Commands { command: SkillsCommands, }, + /// Generate Markdown documentation for the CLI. + #[command(hide = true)] + GenerateDocs, + /// Clean generated artifacts. CleanPyproject { #[arg(name = "KERNEL_DIR")] @@ -345,6 +349,11 @@ fn main() -> Result<()> { validate(kernel_dir)?; Ok(()) } + Commands::GenerateDocs => { + let markdown = clap_markdown::help_markdown::(); + print!("{}", markdown); + Ok(()) + } Commands::Skills { command } => match command { SkillsCommands::Add { skill,