rust(feature): agent skills, prompts, and documentation (#600)

Author: solidiquis

.github/workflows/build-setup.yml

@@ -0,0 +1,7 @@
+# Extra steps injected by cargo-dist into the build job (see
+# `github-build-setup` in dist-workspace.toml). sift_cli's build.rs runs
+# `mdbook build`, so mdbook must be on PATH before `dist build`.
+- name: Install mdbook
+  uses: peaceiris/actions-mdbook@v2
+  with:
+    mdbook-version: '0.4.40'

.github/workflows/rust_ci.yaml

@@ -60,6 +60,11 @@ jobs:
         with:
           components: clippy, rustfmt
 
+      - name: Install mdbook
+        uses: peaceiris/actions-mdbook@v2
+        with:
+          mdbook-version: "0.4.40"
+
       - name: Run cargo check
         run: cargo check --all-features
 

.github/workflows/sift_cli-v-release.yml

@@ -56,17 +56,17 @@ jobs:
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           persist-credentials: false
           submodules: recursive
       - name: Install dist
         # we specify bash to get pipefail; it guards against the `curl` command
         # failing. otherwise `sh` won't catch that `curl` returned non-0
         shell: bash
-        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.30.3/cargo-dist-installer.sh | sh"
+        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.31.0/cargo-dist-installer.sh | sh"
       - name: Cache dist
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: cargo-dist-cache
           path: ~/.cargo/bin/dist
@@ -82,7 +82,7 @@ jobs:
           cat plan-dist-manifest.json
           echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT"
       - name: "Upload dist-manifest.json"
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: artifacts-plan-dist-manifest
           path: plan-dist-manifest.json
@@ -116,7 +116,7 @@ jobs:
       - name: enable windows longpaths
         run: |
           git config --global core.longpaths true
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           persist-credentials: false
           submodules: recursive
@@ -127,11 +127,15 @@ jobs:
             curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
             echo "$HOME/.cargo/bin" >> $GITHUB_PATH
           fi
+      - name: "Install mdbook"
+        uses: "peaceiris/actions-mdbook@v2"
+        with:
+          "mdbook-version": "0.4.40"
       - name: Install dist
         run: ${{ matrix.install_dist.run }}
       # Get the dist-manifest
       - name: Fetch local artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           pattern: artifacts-*
           path: target/distrib/
@@ -158,7 +162,7 @@ jobs:
 
           cp dist-manifest.json "$BUILD_MANIFEST_NAME"
       - name: "Upload artifacts"
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: artifacts-build-local-${{ join(matrix.targets, '_') }}
           path: |
@@ -175,19 +179,19 @@ jobs:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           persist-credentials: false
           submodules: recursive
       - name: Install cached dist
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           name: cargo-dist-cache
           path: ~/.cargo/bin/
       - run: chmod +x ~/.cargo/bin/dist
       # Get all the local artifacts for the global tasks to use (for e.g. checksums)
       - name: Fetch local artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           pattern: artifacts-*
           path: target/distrib/
@@ -205,7 +209,7 @@ jobs:
 
           cp dist-manifest.json "$BUILD_MANIFEST_NAME"
       - name: "Upload artifacts"
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: artifacts-build-global
           path: |
@@ -225,19 +229,19 @@ jobs:
     outputs:
       val: ${{ steps.host.outputs.manifest }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           persist-credentials: false
           submodules: recursive
       - name: Install cached dist
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           name: cargo-dist-cache
           path: ~/.cargo/bin/
       - run: chmod +x ~/.cargo/bin/dist
       # Fetch artifacts from scratch-storage
       - name: Fetch artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           pattern: artifacts-*
           path: target/distrib/
@@ -250,14 +254,14 @@ jobs:
           cat dist-manifest.json
           echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
       - name: "Upload dist-manifest.json"
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           # Overwrite the previous copy
           name: artifacts-dist-manifest
           path: dist-manifest.json
       # Create a GitHub Release while uploading all files to it
       - name: "Download GitHub Artifacts"
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           pattern: artifacts-*
           path: artifacts
@@ -290,7 +294,7 @@ jobs:
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           persist-credentials: false
           submodules: recursive

.gitignore

@@ -366,3 +366,5 @@ cython_debug/
 
 
 rust/crates/**/target/
+rust/crates/sift_cli/assets/docs/book/**
+rust/crates/sift_cli/assets/docs/book-dev/**

Cargo.toml

@@ -31,6 +31,7 @@ arrow-array = "58.3.0"
 arrow-schema = "58.3.0"
 async-channel = "2.2"
 async-trait = "^0.1"
+axum = "0.8"
 mockall = "0.14.0"
 bytes = "1.11.1"
 bytesize = "2"
@@ -48,6 +49,7 @@ futures = { version = "0.3", default-features = false, features = ["alloc"] }
 futures-core = "0.3"
 hyper = { version = "1.8", features = ["server", "http1"] }
 hyper-util = { version = "0.1.20", features = ["service", "server", "tokio"] }
+include_dir = "0.7"
 indicatif = "0.18"
 indoc = "2.0"
 parquet = "58.3.0"
@@ -73,6 +75,7 @@ toml = "0.9"
 tonic = { version = "^0.14", features = ["gzip"] }
 tonic-prost = "^0.14"
 tower = "^0.5"
+tower-serve-static = "0.1"
 tracing = "0.1"
 tracing-appender = "0.2"
 tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }

dist-workspace.toml

@@ -4,7 +4,7 @@ members = ["cargo:"]
 # Config for 'dist'
 [dist]
 # The preferred dist version to use in CI (Cargo.toml SemVer syntax)
-cargo-dist-version = "0.30.3"
+cargo-dist-version = "0.31.0"
 # CI backends to support
 ci = "github"
 # A prefix git tags must include for dist to care about them
@@ -17,3 +17,6 @@ targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-
 install-path = "CARGO_HOME"
 # Whether to install an updater program
 install-updater = false
+# Extra CI steps injected into the build job before `dist build`.
+# sift_cli's build.rs invokes `mdbook build`, so mdbook must be on PATH.
+github-build-setup = "build-setup.yml"

makefile

@@ -1,20 +1,34 @@
 SHELL := /bin/zsh
 
+.PHONY: gen
 gen:
 	bash scripts/gen.sh
 
+.PHONY: gen-go
 gen-go:
 	bash scripts/gen.sh go
 
+.PHONY: gen-python
 gen-python:
 	bash scripts/gen.sh python
 
+.PHONY: gen-rust
 gen-rust:
 	bash scripts/gen.sh rust 
 
+.PHONY: sanitize
 sanitize:
 	echo "Error: sanitize has been removed from this repo. Please see Confluence on proto updating."
 	exit 1
 
+.PHONY: git-hooks
 git-hooks:
 	git config core.hooksPath .githooks
+
+.PHONY: serve-cli-docs
+serve-cli-docs:
+	mdbook serve ./rust/crates/sift_cli/assets/docs -d ./rust/crates/sift_cli/assets/docs/book-dev
+
+.PHONY: build-cli-docs
+build-cli-docs:
+	mdbook build ./rust/crates/sift_cli/assets/docs -d ./rust/crates/sift_cli/assets/docs/book-dev

rust/crates/sift_cli/CLAUDE.md

@@ -0,0 +1,67 @@
+# sift_cli — guidance for Claude
+
+## Agent skill files
+
+`sift-cli` ships two agent-facing instruction files and installs them with
+`sift-cli install agent-skills <agent>`:
+
+- `assets/skills/claude-code/SKILL.md` — Claude Code skill. Has YAML
+  frontmatter (`name`, `description`) and installs to the user's Claude skills
+  directory.
+- `assets/skills/agents-md/AGENTS.md` — the open AGENTS.md standard (Codex,
+  Cursor, Aider, Zed, Windsurf, Amp, Jules, Factory, RooCode). No frontmatter;
+  installs at the project root.
+
+Both are embedded into the binary at build time via `include_str!` in
+`src/cmd/install/agent.rs`. Editing the files changes what the CLI installs, so
+rebuild after any change.
+
+## Keeping the two files in lockstep
+
+The two files exist because Claude Code and the AGENTS.md ecosystem expect
+different on-disk formats. Their **content is the same** and must not drift.
+
+The contract:
+
+1. **The body is shared.** Everything from the `# Sift toolbox` heading to the
+   end of file must be byte-for-byte identical between `SKILL.md` and
+   `AGENTS.md`.
+2. **Only the headers differ.** `SKILL.md` carries YAML frontmatter above the
+   body. `AGENTS.md` has no frontmatter. Both carry the lockstep HTML comment
+   at the top.
+3. **Edit both in the same change.** When you add, remove, or reword anything
+   in the body, apply the identical edit to the other file in the same commit.
+   Never update one alone.
+4. **The frontmatter `description` tracks the body.** When you change which
+   tasks or tools the skill covers, update the `description` in `SKILL.md` so
+   its trigger phrases still match the body.
+
+Verify the bodies match before committing:
+
+```sh
+cd rust/crates/sift_cli/assets/skills
+diff <(awk '/^# Sift toolbox/{f=1} f' claude-code/SKILL.md) \
+     <(awk '/^# Sift toolbox/{f=1} f' agents-md/AGENTS.md)
+```
+
+No output means they are in sync. Any diff must be reconciled before the change
+is done.
+
+## Updating the skill content
+
+Keep the body accurate to the actual tooling. When you change the CLI's
+commands, the MCP server's tools, or the recommended workflow, update the skill
+body to match. In particular:
+
+- The MCP tool list (`list_assets`, `list_runs`, `list_channels`, `get_data`,
+  `sql`, `upload_dataset`) must mirror the tools in the `sift_mcp` crate. If a
+  tool is added or removed there, update both skill files here.
+- The `sift-cli` subcommand list must mirror `src/cli/mod.rs`. If you add an
+  import format or a new subcommand, reflect it in the body.
+- Keep the tool preference order (MCP → `sift-cli` → REST/cURL → Python)
+  intact unless the product guidance itself changes.
+- Keep `sift_client` as the recommended Python module; `sift_py` is deprecated
+  and stays a last resort.
+
+Write in direct voice and keep it concise. The body is read by other agents
+under context pressure, so every line should change what the agent does.

rust/crates/sift_cli/Cargo.toml

@@ -40,6 +40,9 @@ tokio = { workspace = true, features = ["full", "net", "time"] }
 tokio-stream = { workspace = true }
 toml = { workspace = true }
 zip = { workspace = true }
+axum.workspace = true
+tower-serve-static.workspace = true
+include_dir.workspace = true
 
 [dev-dependencies]
 indoc = { workspace = true }

rust/crates/sift_cli/assets/docs/book.toml

@@ -0,0 +1,18 @@
+[book]
+title = "Sift CLI"
+authors = ["solidiquis"]
+language = "en"
+description = "Guide to installing, configuring, and using the Sift CLI."
+
+[output.html]
+default-theme = "navy"
+preferred-dark-theme = "navy"
+git-repository-url = "https://github.com/sift-stack/sift"
+edit-url-template = "https://github.com/sift-stack/sift/edit/main/rust/crates/sift_cli/assets/docs/{path}"
+
+[output.html.fold]
+enable = true
+level = 1
+
+[output.html.search]
+enable = true