diff --git a/.github/workflows/build_docs.yaml b/.github/workflows/build_docs.yaml new file mode 100644 index 0000000000..a0e5ac6413 --- /dev/null +++ b/.github/workflows/build_docs.yaml @@ -0,0 +1,50 @@ +name: Build Docs + +on: + pull_request: + branches: + - main + paths: + - 'docs/**' + - '.github/workflows/build_docs.yaml' + push: + branches: + - main + paths: + - 'docs/**' + - '.github/workflows/build_docs.yaml' + +jobs: + build: + runs-on: ubuntu-latest + defaults: + run: + working-directory: docs + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Use Node.js 22 + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: docs/package-lock.json + - run: npm ci + - run: npm run build + + check: + runs-on: ubuntu-latest + defaults: + run: + working-directory: docs + steps: + - uses: actions/checkout@v4 + - name: Use Node.js 22 + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: docs/package-lock.json + - run: npm ci + - run: npm run check diff --git a/.github/workflows/deploy_docs.yaml b/.github/workflows/deploy_docs.yaml new file mode 100644 index 0000000000..5735149ab5 --- /dev/null +++ b/.github/workflows/deploy_docs.yaml @@ -0,0 +1,59 @@ +name: Deploy to GitHub Pages + +on: + push: + branches: [main] + paths: + - 'docs/**' + - '.github/workflows/deploy_docs.yaml' + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + defaults: + run: + working-directory: docs + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: docs/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Build + run: npm run build + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: ./docs/dist + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000000..38ed2bd578 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,24 @@ +# build output +dist/ +# generated types +.astro/ + +# dependencies +node_modules/ + +# logs +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* + + +# environment variables +.env +.env.production + +# macOS-specific files +.DS_Store + +# vscode +.vscode/ \ No newline at end of file diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..bea6ed42d4 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,88 @@ +# xLLM Documentation + +[English](./README.md) | [简体中文](./README.zh-CN.md) + +This repository contains the Astro + Starlight documentation site for +[xLLM](https://github.com/xLLM-AI/xllm), an LLM inference framework for +high-performance serving on domestic AI accelerators. + +The site is built with Starlight and `starlight-theme-rapide`. It includes a +custom header, bilingual navigation, and a page-level `Copy page` action for +copying documentation content as Markdown. + +## Documentation Structure + +The documentation is maintained in two parallel language trees: + +- English: `src/content/docs/en` +- Simplified Chinese: `src/content/docs/zh` + +The root path redirects to the English documentation: + +- `/` redirects to `/en/` +- `/en/` serves the English documentation +- `/zh/` serves the Simplified Chinese documentation + +When adding or moving pages, keep matching relative paths in both language +trees so Starlight can switch between languages for the same topic. + +## Project Layout + +```text +. +├── astro.config.mjs # Starlight, locale, sidebar, and component config +├── package.json # npm scripts and dependencies +├── src/ +│ ├── assets/ # Site-level assets such as the logo +│ ├── components/ # Starlight component overrides +│ ├── content/ +│ │ └── docs/ +│ │ ├── en/ # English documentation +│ │ ├── zh/ # Simplified Chinese documentation +│ │ └── assets/ # Documentation images and diagrams +│ ├── pages/index.astro # Redirect from / to /en/ +│ └── styles/theme.css # Project theme customizations +└── public/ # Static public assets +``` + +## Local Development + +Install dependencies: + +```sh +npm install +``` + +Start the local development server: + +```sh +npm run dev +``` + +Build the production site: + +```sh +npm run build +``` + +Preview the production build: + +```sh +npm run preview +``` + +## Editing Documentation + +- Put user-facing content under `src/content/docs/en` and + `src/content/docs/zh`. +- Keep English and Chinese files aligned by path when a page exists in both + languages. +- Store shared documentation images in `src/content/docs/assets`. +- Update the `sidebar` section in `astro.config.mjs` when adding new sections + that should appear in navigation. +- Run `npm run build` before submitting changes to catch broken routes, + frontmatter errors, and Starlight content issues. + +## Related Repository + +- xLLM source code: diff --git a/docs/README_zh.md b/docs/README_zh.md new file mode 100644 index 0000000000..0ccbc1bb50 --- /dev/null +++ b/docs/README_zh.md @@ -0,0 +1,84 @@ +# xLLM 文档站 + +[English](./README.md) | [简体中文](./README.zh-CN.md) + +本仓库是 [xLLM](https://github.com/xLLM-AI/xllm) 的 Astro + Starlight +文档站。xLLM 是面向国产 AI 加速器的高性能大语言模型推理框架。 + +站点基于 Starlight 和 `starlight-theme-rapide` 构建，包含自定义顶部导航、 +中英文切换，以及用于将文档正文复制为 Markdown 的 `Copy page` 页面操作。 + +## 文档结构 + +文档内容按语言维护在两个并行目录中： + +- 英文：`src/content/docs/en` +- 简体中文：`src/content/docs/zh` + +当前站点路由规则如下： + +- `/` 重定向到 `/en/` +- `/en/` 对应英文文档 +- `/zh/` 对应简体中文文档 + +新增或移动页面时，请尽量在两个语言目录中保持相同的相对路径，方便 Starlight +在同一主题的不同语言版本之间切换。 + +## 项目结构 + +```text +. +├── astro.config.mjs # Starlight、语言、侧边栏和组件配置 +├── package.json # npm 脚本和依赖 +├── src/ +│ ├── assets/ # 站点级资源，例如 logo +│ ├── components/ # Starlight 组件覆盖 +│ ├── content/ +│ │ └── docs/ +│ │ ├── en/ # 英文文档 +│ │ ├── zh/ # 简体中文文档 +│ │ └── assets/ # 文档图片和架构图 +│ ├── pages/index.astro # 从 / 重定向到 /en/ +│ └── styles/theme.css # 项目主题样式 +└── public/ # 静态公共资源 +``` + +## 本地开发 + +安装依赖： + +```sh +npm install +``` + +启动本地开发服务： + +```sh +npm run dev +``` + +构建生产站点： + +```sh +npm run build +``` + +预览生产构建结果： + +```sh +npm run preview +``` + +## 编辑文档 + +- 面向用户的文档内容放在 `src/content/docs/en` 和 `src/content/docs/zh`。 +- 同一个页面同时存在中英文版本时，尽量保持两个文件的相对路径一致。 +- 共享的文档图片放在 `src/content/docs/assets`。 +- 新增需要出现在导航中的章节时，更新 `astro.config.mjs` 中的 `sidebar` + 配置。 +- 提交前运行 `npm run build`，用于检查路由、frontmatter 和 Starlight + 内容问题。 + +## 相关仓库 + +- xLLM 源码： diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs new file mode 100644 index 0000000000..1452f3200d --- /dev/null +++ b/docs/astro.config.mjs @@ -0,0 +1,340 @@ +// @ts-check +import { defineConfig } from 'astro/config'; +import starlight from '@astrojs/starlight'; +import starlightThemeRapide from 'starlight-theme-rapide'; +import rehypeSimpleIcons from './src/plugins/rehypeSimpleIcons.mjs'; + +// https://astro.build/config +export default defineConfig({ + markdown: { + rehypePlugins: [rehypeSimpleIcons], + }, + integrations: [ + starlight({ + plugins: [starlightThemeRapide()], + customCss: ['./src/styles/theme.css'], + lastUpdated: true, + components: { + Head: './src/components/Head.astro', + Header: './src/components/Header.astro', + PageTitle: './src/components/PageTitle.astro', + SiteTitle: './src/components/SiteTitle.astro', + }, + title: { + en: 'xLLM', + 'zh-CN': 'xLLM', + }, + locales: { + en: { + label: 'EN', + lang: 'en', + }, + zh: { + label: '中', + lang: 'zh-CN', + }, + }, + defaultLocale: 'en', + logo: { + src: './src/assets/logo_with_llm.png', + alt: 'xLLM', + replacesTitle: true, + }, + social: [{ icon: 'github', label: 'GitHub', href: 'https://github.com/xLLM-AI/xllm' }], + sidebar: [ + { label: 'Home', translations: { 'zh-CN': '主页' }, slug: '' }, + { + label: 'Getting Started', + translations: { 'zh-CN': '开始使用' }, + items: [ + 'getting_started/quick_start', + 'getting_started/launch_xllm', + 'getting_started/multi_machine', + 'getting_started/online_service', + 'getting_started/offline_service', + { + label: 'Supported Models', + translations: { 'zh-CN': '模型支持列表' }, + slug: 'supported_models', + }, + ], + }, + { + label: 'Hardware', + translations: { 'zh-CN': '硬件' }, + items: [ + { + label: 'Hardware Platforms', + translations: { 'zh-CN': '硬件平台' }, + slug: 'hardware/overview', + }, + { + label: 'NVIDIA GPU', + translations: { 'zh-CN': 'NVIDIA GPU' }, + slug: 'hardware/nvidia_gpu', + }, + { + label: 'Ascend NPU', + translations: { 'zh-CN': '昇腾 NPU' }, + slug: 'hardware/ascend_npu', + }, + { + label: 'Cambricon MLU', + translations: { 'zh-CN': '寒武纪 MLU' }, + slug: 'hardware/cambricon_mlu', + }, + { + label: 'Hygon DCU', + translations: { 'zh-CN': '海光 DCU' }, + slug: 'hardware/dcu', + }, + { + label: 'MetaX MACA', + translations: { 'zh-CN': '沐曦 MACA' }, + slug: 'hardware/metax_maca', + }, + { + label: 'Mthreads MUSA', + translations: { 'zh-CN': '摩尔线程 MUSA' }, + slug: 'hardware/musa', + }, + ], + }, + { + label: 'User Guide', + translations: { 'zh-CN': '用户指南' }, + items: [ + { + label: 'Advanced Features', + translations: { 'zh-CN': '高级功能' }, + autogenerate: { directory: 'features' }, + }, + ], + }, + { + label: 'CookBook', + translations: { 'zh-CN': '实践指南' }, + items: [ + { + label: 'Autoregressive Models', + translations: { 'zh-CN': '自回归模型' }, + collapsed: false, + items: [ + { + label: 'Qwen', + translations: { 'zh-CN': 'Qwen' }, + collapsed: true, + items: [ + { + label: 'Qwen3.5', + translations: { 'zh-CN': 'Qwen3.5' }, + slug: 'cookbook/autoregressive_models/qwen/qwen3_5', + }, + { + label: 'Qwen3', + translations: { 'zh-CN': 'Qwen3' }, + slug: 'cookbook/autoregressive_models/qwen/qwen3', + }, + { + label: 'Qwen3-Next', + translations: { 'zh-CN': 'Qwen3-Next' }, + slug: 'cookbook/autoregressive_models/qwen/qwen3_next', + }, + { + label: 'Qwen3-VL', + translations: { 'zh-CN': 'Qwen3-VL' }, + slug: 'cookbook/autoregressive_models/qwen/qwen3_vl', + }, + { + label: 'Qwen2.5-VL', + translations: { 'zh-CN': 'Qwen2.5-VL' }, + slug: 'cookbook/autoregressive_models/qwen/qwen2_5_vl', + }, + ], + }, + { + label: 'DeepSeek', + translations: { 'zh-CN': 'DeepSeek' }, + collapsed: true, + items: [ + { + label: 'DeepSeek-V4', + translations: { 'zh-CN': 'DeepSeek-V4' }, + slug: 'cookbook/autoregressive_models/deepseek/deepseek_v4', + }, + { + label: 'DeepSeek-V3.2', + translations: { 'zh-CN': 'DeepSeek-V3.2' }, + slug: 'cookbook/autoregressive_models/deepseek/deepseek_v3_2', + }, + { + label: 'DeepSeek-V3.1', + translations: { 'zh-CN': 'DeepSeek-V3.1' }, + slug: 'cookbook/autoregressive_models/deepseek/deepseek_v3_1', + }, + { + label: 'DeepSeek-V3', + translations: { 'zh-CN': 'DeepSeek-V3' }, + slug: 'cookbook/autoregressive_models/deepseek/deepseek_v3', + }, + { + label: 'DeepSeek-R1', + translations: { 'zh-CN': 'DeepSeek-R1' }, + slug: 'cookbook/autoregressive_models/deepseek/deepseek_r1', + }, + ], + }, + { + label: 'GLM', + translations: { 'zh-CN': 'GLM' }, + collapsed: true, + items: [ + { + label: 'GLM-5.2', + translations: { 'zh-CN': 'GLM-5.2' }, + slug: 'cookbook/autoregressive_models/glm/glm_5', + }, + { + label: 'GLM-5.1', + translations: { 'zh-CN': 'GLM-5.1' }, + slug: 'cookbook/autoregressive_models/glm/glm_5', + }, + { + label: 'GLM-5', + translations: { 'zh-CN': 'GLM-5' }, + slug: 'cookbook/autoregressive_models/glm/glm_5', + }, + { + label: 'GLM-4.7', + translations: { 'zh-CN': 'GLM-4.7' }, + slug: 'cookbook/autoregressive_models/glm/glm_4_7', + }, + { + label: 'GLM-4.7-Flash', + translations: { 'zh-CN': 'GLM-4.7-Flash' }, + slug: 'cookbook/autoregressive_models/glm/glm_4_7_flash', + }, + { + label: 'GLM-4.6', + translations: { 'zh-CN': 'GLM-4.6' }, + slug: 'cookbook/autoregressive_models/glm/glm_4_6', + }, + { + label: 'GLM-4.6V', + translations: { 'zh-CN': 'GLM-4.6V' }, + slug: 'cookbook/autoregressive_models/glm/glm_4_6v', + }, + { + label: 'GLM-4.5', + translations: { 'zh-CN': 'GLM-4.5' }, + slug: 'cookbook/autoregressive_models/glm/glm_4_5', + }, + { + label: 'GLM-4.5V', + translations: { 'zh-CN': 'GLM-4.5V' }, + slug: 'cookbook/autoregressive_models/glm/glm_4_5v', + }, + ], + }, + { + label: 'Kimi', + translations: { 'zh-CN': 'Kimi' }, + collapsed: true, + items: [ + { + label: 'Kimi2', + translations: { 'zh-CN': 'Kimi2' }, + slug: 'cookbook/autoregressive_models/kimi/kimi2', + }, + { + label: 'Kimi-K2.5 / Kimi-K2.6', + translations: { 'zh-CN': 'Kimi-K2.5 / Kimi-K2.6' }, + slug: 'cookbook/autoregressive_models/kimi/kimi2_5', + }, + ], + }, + { + label: 'MinMax', + translations: { 'zh-CN': 'MinMax' }, + collapsed: true, + items: [ + { + label: 'MiniMax-M2.7', + translations: { 'zh-CN': 'MiniMax-M2.7' }, + slug: 'cookbook/autoregressive_models/minmax/minmax_m2_7', + }, + ], + }, + ], + }, + { + label: 'Diffusion Models', + translations: { 'zh-CN': '扩散模型' }, + collapsed: false, + items: [ + { + label: 'Flux', + translations: { 'zh-CN': 'Flux' }, + collapsed: true, + items: [ + { + label: 'Flux', + translations: { 'zh-CN': 'Flux' }, + slug: 'cookbook/diffusion_models/flux/flux', + }, + { + label: 'Flux2', + translations: { 'zh-CN': 'Flux2' }, + slug: 'cookbook/diffusion_models/flux/flux2', + }, + ], + }, + { + label: 'Wan', + translations: { 'zh-CN': 'Wan' }, + collapsed: true, + items: [ + { + label: 'Wan2.1', + translations: { 'zh-CN': 'Wan2.1' }, + slug: 'cookbook/diffusion_models/wan/wan2_1', + }, + ], + }, + { + label: 'Qwen-Image', + translations: { 'zh-CN': 'Qwen-Image' }, + collapsed: true, + items: [ + { + label: 'Qwen-Image', + translations: { 'zh-CN': 'Qwen-Image' }, + slug: 'cookbook/diffusion_models/qwen_image/qwen_image', + }, + ], + }, + ], + }, + ], + }, + { + label: 'Developer Guide', + translations: { 'zh-CN': '开发者指南' }, + items: [ + { + label: 'Development', + translations: { 'zh-CN': '开发' }, + autogenerate: { directory: 'dev_guide' }, + }, + { + label: 'Design', + translations: { 'zh-CN': '设计文档' }, + autogenerate: { directory: 'design' }, + }, + ], + }, + { label: 'CLI Reference', translations: { 'zh-CN': 'CLI 参考' }, slug: 'cli_reference' }, + ], + }), + ], +}); diff --git a/docs/package-lock.json b/docs/package-lock.json new file mode 100644 index 0000000000..d50325a051 --- /dev/null +++ b/docs/package-lock.json @@ -0,0 +1,7301 @@ +{ + "name": "awesome-aurora", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "awesome-aurora", + "version": "0.0.1", + "dependencies": { + "@astrojs/starlight": "^0.38.4", + "astro": "^6.1.9", + "sharp": "^0.34.5", + "starlight-theme-rapide": "^0.5.2" + }, + "devDependencies": { + "@astrojs/check": "^0.9.9", + "typescript": "^5.9.3" + } + }, + "node_modules/@astrojs/check": { + "version": "0.9.9", + "resolved": "https://registry.npmjs.org/@astrojs/check/-/check-0.9.9.tgz", + "integrity": "sha512-A5UW8uIuErLWEoRQvzgXpO1gTjUFtK8r7nU2Z7GewAMxUb7bPvpk11qaKKgxqXlHJWlAvaaxy+Xg28A6bmQ1Tg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@astrojs/language-server": "^2.16.7", + "chokidar": "^4.0.3", + "kleur": "^4.1.5", + "yargs": "^17.7.2" + }, + "bin": { + "astro-check": "bin/astro-check.js" + }, + "peerDependencies": { + "typescript": "^5.0.0 || ^6.0.0" + } + }, + "node_modules/@astrojs/check/node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@astrojs/check/node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@astrojs/compiler": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@astrojs/compiler/-/compiler-3.0.1.tgz", + "integrity": "sha512-z97oYbdebO5aoWzuJ/8q5hLK232+17KcLZ7cJ8BCWk6+qNzVxn/gftC0KzMBUTD8WAaBkPpNSQK6PXLnNrZ0CA==", + "license": "MIT" + }, + "node_modules/@astrojs/internal-helpers": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@astrojs/internal-helpers/-/internal-helpers-0.9.0.tgz", + "integrity": "sha512-GdYkzR26re8izmyYlBqf4z2s7zNngmWLFuxw0UKiPNqHraZGS6GKWIwSHgS22RDlu2ePFJ8bzmpBcUszut/SDg==", + "license": "MIT", + "dependencies": { + "picomatch": "^4.0.4" + } + }, + "node_modules/@astrojs/language-server": { + "version": "2.16.10", + "resolved": "https://registry.npmjs.org/@astrojs/language-server/-/language-server-2.16.10.tgz", + "integrity": "sha512-87VQ/5GSdHlRnUA+hGuerYyIGAj+9RbZmATyuKLEUePinUXhQ5YkRnRrHhOD9sSi5JOErLjrLkHnfZFEvGrV8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@astrojs/compiler": "^2.13.1", + "@astrojs/yaml2ts": "^0.2.4", + "@jridgewell/sourcemap-codec": "^1.5.5", + "@volar/kit": "~2.4.28", + "@volar/language-core": "~2.4.28", + "@volar/language-server": "~2.4.28", + "@volar/language-service": "~2.4.28", + "muggle-string": "^0.4.1", + "tinyglobby": "^0.2.16", + "volar-service-css": "0.0.70", + "volar-service-emmet": "0.0.70", + "volar-service-html": "0.0.70", + "volar-service-prettier": "0.0.70", + "volar-service-typescript": "0.0.70", + "volar-service-typescript-twoslash-queries": "0.0.70", + "volar-service-yaml": "0.0.70", + "vscode-html-languageservice": "^5.6.2", + "vscode-uri": "^3.1.0" + }, + "bin": { + "astro-ls": "bin/nodeServer.js" + }, + "peerDependencies": { + "prettier": "^3.0.0", + "prettier-plugin-astro": ">=0.11.0" + }, + "peerDependenciesMeta": { + "prettier": { + "optional": true + }, + "prettier-plugin-astro": { + "optional": true + } + } + }, + "node_modules/@astrojs/language-server/node_modules/@astrojs/compiler": { + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/@astrojs/compiler/-/compiler-2.13.1.tgz", + "integrity": "sha512-f3FN83d2G/v32ipNClRKgYv30onQlMZX1vCeZMjPsMMPl1mDpmbl0+N5BYo4S/ofzqJyS5hvwacEo0CCVDn/Qg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@astrojs/markdown-remark": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@astrojs/markdown-remark/-/markdown-remark-7.1.1.tgz", + "integrity": "sha512-C6e9BnLGlbdv6bV8MYGeHpHxsUHrCrB4OuRLqi5LI7oiBVcBcqfUN06zpwFQdHgV48QCCrMmLpyqBr7VqC+swA==", + "license": "MIT", + "dependencies": { + "@astrojs/internal-helpers": "0.9.0", + "@astrojs/prism": "4.0.1", + "github-slugger": "^2.0.0", + "hast-util-from-html": "^2.0.3", + "hast-util-to-text": "^4.0.2", + "js-yaml": "^4.1.1", + "mdast-util-definitions": "^6.0.0", + "rehype-raw": "^7.0.0", + "rehype-stringify": "^10.0.1", + "remark-gfm": "^4.0.1", + "remark-parse": "^11.0.0", + "remark-rehype": "^11.1.2", + "remark-smartypants": "^3.0.2", + "retext-smartypants": "^6.2.0", + "shiki": "^4.0.0", + "smol-toml": "^1.6.0", + "unified": "^11.0.5", + "unist-util-remove-position": "^5.0.0", + "unist-util-visit": "^5.1.0", + "unist-util-visit-parents": "^6.0.2", + "vfile": "^6.0.3" + } + }, + "node_modules/@astrojs/mdx": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@astrojs/mdx/-/mdx-5.0.4.tgz", + "integrity": "sha512-tSbuuYueNODiFAFaME7pjHY5lOLoxBYJi1cKd6scw9+a4ZO7C7UGdafEoVAQvOV2eO8a6RaHSAJYGVPL1w8BPA==", + "license": "MIT", + "dependencies": { + "@astrojs/markdown-remark": "7.1.1", + "@mdx-js/mdx": "^3.1.1", + "acorn": "^8.16.0", + "es-module-lexer": "^2.0.0", + "estree-util-visit": "^2.0.0", + "hast-util-to-html": "^9.0.5", + "piccolore": "^0.1.3", + "rehype-raw": "^7.0.0", + "remark-gfm": "^4.0.1", + "remark-smartypants": "^3.0.2", + "source-map": "^0.7.6", + "unist-util-visit": "^5.1.0", + "vfile": "^6.0.3" + }, + "engines": { + "node": ">=22.12.0" + }, + "peerDependencies": { + "astro": "^6.0.0" + } + }, + "node_modules/@astrojs/prism": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@astrojs/prism/-/prism-4.0.1.tgz", + "integrity": "sha512-nksZQVjlferuWzhPsBpQ1JE5XuKAf1id1/9Hj4a9KG4+ofrlzxUUwX4YGQF/SuDiuiGKEnzopGOt38F3AnVWsQ==", + "license": "MIT", + "dependencies": { + "prismjs": "^1.30.0" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/@astrojs/sitemap": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/@astrojs/sitemap/-/sitemap-3.7.2.tgz", + "integrity": "sha512-PqkzkcZTb5ICiyIR8VoKbIAP/laNRXi5tw616N1Ckk+40oNB8Can1AzVV56lrbC5GKSZFCyJYUVYqVivMisvpA==", + "license": "MIT", + "dependencies": { + "sitemap": "^9.0.0", + "stream-replace-string": "^2.0.0", + "zod": "^4.3.6" + } + }, + "node_modules/@astrojs/starlight": { + "version": "0.38.4", + "resolved": "https://registry.npmjs.org/@astrojs/starlight/-/starlight-0.38.4.tgz", + "integrity": "sha512-TGFIr2aVC+gcZCPQzJOO4ZnA/yL3jRnsUDcKlVdEhxhxaOQnWr9lZ9MRScg9zU6uh3HVeZAmmjkLCdTlHdcaZA==", + "license": "MIT", + "dependencies": { + "@astrojs/markdown-remark": "^7.0.0", + "@astrojs/mdx": "^5.0.0", + "@astrojs/sitemap": "^3.7.1", + "@pagefind/default-ui": "^1.3.0", + "@types/hast": "^3.0.4", + "@types/js-yaml": "^4.0.9", + "@types/mdast": "^4.0.4", + "astro-expressive-code": "^0.41.6", + "bcp-47": "^2.1.0", + "hast-util-from-html": "^2.0.1", + "hast-util-select": "^6.0.2", + "hast-util-to-string": "^3.0.0", + "hastscript": "^9.0.0", + "i18next": "^23.11.5", + "js-yaml": "^4.1.0", + "klona": "^2.0.6", + "magic-string": "^0.30.17", + "mdast-util-directive": "^3.0.0", + "mdast-util-to-markdown": "^2.1.0", + "mdast-util-to-string": "^4.0.0", + "pagefind": "^1.3.0", + "rehype": "^13.0.1", + "rehype-format": "^5.0.0", + "remark-directive": "^3.0.0", + "ultrahtml": "^1.6.0", + "unified": "^11.0.5", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.2" + }, + "peerDependencies": { + "astro": "^6.0.0" + } + }, + "node_modules/@astrojs/telemetry": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@astrojs/telemetry/-/telemetry-3.3.1.tgz", + "integrity": "sha512-7fcIxXS9J4ls5tr8b3ww9rbAIz2+HrhNJYZdkAhhB4za/I5IZ/60g+Bs8q7zwG0tOIZfNB4JWhVJ1Qkl/OrNCw==", + "license": "MIT", + "dependencies": { + "ci-info": "^4.4.0", + "dlv": "^1.1.3", + "dset": "^3.1.4", + "is-docker": "^4.0.0", + "is-wsl": "^3.1.1", + "which-pm-runs": "^1.1.0" + }, + "engines": { + "node": "18.20.8 || ^20.3.0 || >=22.0.0" + } + }, + "node_modules/@astrojs/yaml2ts": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@astrojs/yaml2ts/-/yaml2ts-0.2.4.tgz", + "integrity": "sha512-8oddpOae35pJsXPQXhTkM0ypfKPskVsh2bCxRtbf7e+/Epw2nReakFYpLKjZMEr75CsoF203PMnCocpfz0s69A==", + "dev": true, + "license": "MIT", + "dependencies": { + "yaml": "^2.8.3" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.2.tgz", + "integrity": "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", + "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", + "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@capsizecss/unpack": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@capsizecss/unpack/-/unpack-4.0.0.tgz", + "integrity": "sha512-VERIM64vtTP1C4mxQ5thVT9fK0apjPFobqybMtA1UdUujWka24ERHbRHFGmpbbhp73MhV+KSsHQH9C6uOTdEQA==", + "license": "MIT", + "dependencies": { + "fontkitten": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@clack/core": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@clack/core/-/core-1.2.0.tgz", + "integrity": "sha512-qfxof/3T3t9DPU/Rj3OmcFyZInceqj/NVtO9rwIuJqCUgh32gwPjpFQQp/ben07qKlhpwq7GzfWpST4qdJ5Drg==", + "license": "MIT", + "dependencies": { + "fast-wrap-ansi": "^0.1.3", + "sisteransi": "^1.0.5" + } + }, + "node_modules/@clack/prompts": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@clack/prompts/-/prompts-1.2.0.tgz", + "integrity": "sha512-4jmztR9fMqPMjz6H/UZXj0zEmE43ha1euENwkckKKel4XpSfokExPo5AiVStdHSAlHekz4d0CA/r45Ok1E4D3w==", + "license": "MIT", + "dependencies": { + "@clack/core": "1.2.0", + "fast-string-width": "^1.1.0", + "fast-wrap-ansi": "^0.1.3", + "sisteransi": "^1.0.5" + } + }, + "node_modules/@ctrl/tinycolor": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@ctrl/tinycolor/-/tinycolor-4.2.0.tgz", + "integrity": "sha512-kzyuwOAQnXJNLS9PSyrk0CWk35nWJW/zl/6KvnTBMFK65gm7U1/Z5BqjxeapjZCIhQcM/DsrEmcbRwDyXyXK4A==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@emmetio/abbreviation": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/@emmetio/abbreviation/-/abbreviation-2.3.3.tgz", + "integrity": "sha512-mgv58UrU3rh4YgbE/TzgLQwJ3pFsHHhCLqY20aJq+9comytTXUDNGG/SMtSeMJdkpxgXSXunBGLD8Boka3JyVA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@emmetio/scanner": "^1.0.4" + } + }, + "node_modules/@emmetio/css-abbreviation": { + "version": "2.1.8", + "resolved": "https://registry.npmjs.org/@emmetio/css-abbreviation/-/css-abbreviation-2.1.8.tgz", + "integrity": "sha512-s9yjhJ6saOO/uk1V74eifykk2CBYi01STTK3WlXWGOepyKa23ymJ053+DNQjpFcy1ingpaO7AxCcwLvHFY9tuw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@emmetio/scanner": "^1.0.4" + } + }, + "node_modules/@emmetio/css-parser": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@emmetio/css-parser/-/css-parser-0.4.1.tgz", + "integrity": "sha512-2bC6m0MV/voF4CTZiAbG5MWKbq5EBmDPKu9Sb7s7nVcEzNQlrZP6mFFFlIaISM8X6514H9shWMme1fCm8cWAfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@emmetio/stream-reader": "^2.2.0", + "@emmetio/stream-reader-utils": "^0.1.0" + } + }, + "node_modules/@emmetio/html-matcher": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@emmetio/html-matcher/-/html-matcher-1.3.0.tgz", + "integrity": "sha512-NTbsvppE5eVyBMuyGfVu2CRrLvo7J4YHb6t9sBFLyY03WYhXET37qA4zOYUjBWFCRHO7pS1B9khERtY0f5JXPQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "@emmetio/scanner": "^1.0.0" + } + }, + "node_modules/@emmetio/scanner": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@emmetio/scanner/-/scanner-1.0.4.tgz", + "integrity": "sha512-IqRuJtQff7YHHBk4G8YZ45uB9BaAGcwQeVzgj/zj8/UdOhtQpEIupUhSk8dys6spFIWVZVeK20CzGEnqR5SbqA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@emmetio/stream-reader": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@emmetio/stream-reader/-/stream-reader-2.2.0.tgz", + "integrity": "sha512-fXVXEyFA5Yv3M3n8sUGT7+fvecGrZP4k6FnWWMSZVQf69kAq0LLpaBQLGcPR30m3zMmKYhECP4k/ZkzvhEW5kw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@emmetio/stream-reader-utils": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@emmetio/stream-reader-utils/-/stream-reader-utils-0.1.0.tgz", + "integrity": "sha512-ZsZ2I9Vzso3Ho/pjZFsmmZ++FWeEd/txqybHTm4OgaZzdS8V9V/YYWQwg5TC38Z7uLWUV1vavpLLbjJtKubR1A==", + "dev": true, + "license": "MIT" + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@expressive-code/core": { + "version": "0.41.7", + "resolved": "https://registry.npmjs.org/@expressive-code/core/-/core-0.41.7.tgz", + "integrity": "sha512-ck92uZYZ9Wba2zxkiZLsZGi9N54pMSAVdrI9uW3Oo9AtLglD5RmrdTwbYPCT2S/jC36JGB2i+pnQtBm/Ib2+dg==", + "license": "MIT", + "dependencies": { + "@ctrl/tinycolor": "^4.0.4", + "hast-util-select": "^6.0.2", + "hast-util-to-html": "^9.0.1", + "hast-util-to-text": "^4.0.1", + "hastscript": "^9.0.0", + "postcss": "^8.4.38", + "postcss-nested": "^6.0.1", + "unist-util-visit": "^5.0.0", + "unist-util-visit-parents": "^6.0.1" + } + }, + "node_modules/@expressive-code/plugin-frames": { + "version": "0.41.7", + "resolved": "https://registry.npmjs.org/@expressive-code/plugin-frames/-/plugin-frames-0.41.7.tgz", + "integrity": "sha512-diKtxjQw/979cTglRFaMCY/sR6hWF0kSMg8jsKLXaZBSfGS0I/Hoe7Qds3vVEgeoW+GHHQzMcwvgx/MOIXhrTA==", + "license": "MIT", + "dependencies": { + "@expressive-code/core": "^0.41.7" + } + }, + "node_modules/@expressive-code/plugin-shiki": { + "version": "0.41.7", + "resolved": "https://registry.npmjs.org/@expressive-code/plugin-shiki/-/plugin-shiki-0.41.7.tgz", + "integrity": "sha512-DL605bLrUOgqTdZ0Ot5MlTaWzppRkzzqzeGEu7ODnHF39IkEBbFdsC7pbl3LbUQ1DFtnfx6rD54k/cdofbW6KQ==", + "license": "MIT", + "dependencies": { + "@expressive-code/core": "^0.41.7", + "shiki": "^3.2.2" + } + }, + "node_modules/@expressive-code/plugin-shiki/node_modules/@shikijs/core": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/@shikijs/core/-/core-3.23.0.tgz", + "integrity": "sha512-NSWQz0riNb67xthdm5br6lAkvpDJRTgB36fxlo37ZzM2yq0PQFFzbd8psqC2XMPgCzo1fW6cVi18+ArJ44wqgA==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.23.0", + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.4", + "hast-util-to-html": "^9.0.5" + } + }, + "node_modules/@expressive-code/plugin-shiki/node_modules/@shikijs/engine-javascript": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/@shikijs/engine-javascript/-/engine-javascript-3.23.0.tgz", + "integrity": "sha512-aHt9eiGFobmWR5uqJUViySI1bHMqrAgamWE1TYSUoftkAeCCAiGawPMwM+VCadylQtF4V3VNOZ5LmfItH5f3yA==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.23.0", + "@shikijs/vscode-textmate": "^10.0.2", + "oniguruma-to-es": "^4.3.4" + } + }, + "node_modules/@expressive-code/plugin-shiki/node_modules/@shikijs/engine-oniguruma": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/@shikijs/engine-oniguruma/-/engine-oniguruma-3.23.0.tgz", + "integrity": "sha512-1nWINwKXxKKLqPibT5f4pAFLej9oZzQTsby8942OTlsJzOBZ0MWKiwzMsd+jhzu8YPCHAswGnnN1YtQfirL35g==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.23.0", + "@shikijs/vscode-textmate": "^10.0.2" + } + }, + "node_modules/@expressive-code/plugin-shiki/node_modules/@shikijs/langs": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/@shikijs/langs/-/langs-3.23.0.tgz", + "integrity": "sha512-2Ep4W3Re5aB1/62RSYQInK9mM3HsLeB91cHqznAJMuylqjzNVAVCMnNWRHFtcNHXsoNRayP9z1qj4Sq3nMqYXg==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.23.0" + } + }, + "node_modules/@expressive-code/plugin-shiki/node_modules/@shikijs/themes": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/@shikijs/themes/-/themes-3.23.0.tgz", + "integrity": "sha512-5qySYa1ZgAT18HR/ypENL9cUSGOeI2x+4IvYJu4JgVJdizn6kG4ia5Q1jDEOi7gTbN4RbuYtmHh0W3eccOrjMA==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.23.0" + } + }, + "node_modules/@expressive-code/plugin-shiki/node_modules/@shikijs/types": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-3.23.0.tgz", + "integrity": "sha512-3JZ5HXOZfYjsYSk0yPwBrkupyYSLpAE26Qc0HLghhZNGTZg/SKxXIIgoxOpmmeQP0RRSDJTk1/vPfw9tbw+jSQ==", + "license": "MIT", + "dependencies": { + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.4" + } + }, + "node_modules/@expressive-code/plugin-shiki/node_modules/shiki": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-3.23.0.tgz", + "integrity": "sha512-55Dj73uq9ZXL5zyeRPzHQsK7Nbyt6Y10k5s7OjuFZGMhpp4r/rsLBH0o/0fstIzX1Lep9VxefWljK/SKCzygIA==", + "license": "MIT", + "dependencies": { + "@shikijs/core": "3.23.0", + "@shikijs/engine-javascript": "3.23.0", + "@shikijs/engine-oniguruma": "3.23.0", + "@shikijs/langs": "3.23.0", + "@shikijs/themes": "3.23.0", + "@shikijs/types": "3.23.0", + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.4" + } + }, + "node_modules/@expressive-code/plugin-text-markers": { + "version": "0.41.7", + "resolved": "https://registry.npmjs.org/@expressive-code/plugin-text-markers/-/plugin-text-markers-0.41.7.tgz", + "integrity": "sha512-Ewpwuc5t6eFdZmWlFyeuy3e1PTQC0jFvw2Q+2bpcWXbOZhPLsT7+h8lsSIJxb5mS7wZko7cKyQ2RLYDyK6Fpmw==", + "license": "MIT", + "dependencies": { + "@expressive-code/core": "^0.41.7" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "license": "MIT" + }, + "node_modules/@mdx-js/mdx": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@mdx-js/mdx/-/mdx-3.1.1.tgz", + "integrity": "sha512-f6ZO2ifpwAQIpzGWaBQT2TXxPv6z3RBzQKpVftEWN78Vl/YweF1uwussDx8ECAXVtr3Rs89fKyG9YlzUs9DyGQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdx": "^2.0.0", + "acorn": "^8.0.0", + "collapse-white-space": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "estree-util-scope": "^1.0.0", + "estree-walker": "^3.0.0", + "hast-util-to-jsx-runtime": "^2.0.0", + "markdown-extensions": "^2.0.0", + "recma-build-jsx": "^1.0.0", + "recma-jsx": "^1.0.0", + "recma-stringify": "^1.0.0", + "rehype-recma": "^1.0.0", + "remark-mdx": "^3.0.0", + "remark-parse": "^11.0.0", + "remark-rehype": "^11.0.0", + "source-map": "^0.7.0", + "unified": "^11.0.0", + "unist-util-position-from-estree": "^2.0.0", + "unist-util-stringify-position": "^4.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/@oslojs/encoding": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@oslojs/encoding/-/encoding-1.1.0.tgz", + "integrity": "sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==", + "license": "MIT" + }, + "node_modules/@pagefind/darwin-arm64": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/darwin-arm64/-/darwin-arm64-1.5.2.tgz", + "integrity": "sha512-MXpI+7HsAdPkvJ0gk9xj9g541BCqBZOBbdwj9g6lB5LCj6kSV6nqDSjzcAJwvOsfu0fjwvC8hQU+ecfhp+MpiQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@pagefind/darwin-x64": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/darwin-x64/-/darwin-x64-1.5.2.tgz", + "integrity": "sha512-IojxFWMEJe0RQ7PQ3KXQsPIImNsbpPYpoZ+QUDrL8fAl/O27IX+LVLs74/UzEZy5uA2LD8Nz1AiwKr72vrkZQw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@pagefind/default-ui": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/default-ui/-/default-ui-1.5.2.tgz", + "integrity": "sha512-pm1LMnQg8N2B3n2TnjKlhaFihpz6zTiA4HiGQ6/slKO/+8K9CAU5kcjdSSPgpuk1PMuuN4hxLipUIifnrkl3Sg==", + "license": "MIT" + }, + "node_modules/@pagefind/freebsd-x64": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/freebsd-x64/-/freebsd-x64-1.5.2.tgz", + "integrity": "sha512-7EVzo9+0w+2cbe671BtMj10UlNo83I+HrLVLfRxO731svHRJKUfJ/mo05gU14pe9PCfpKNQT8FS3Xc/oDN6pOA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@pagefind/linux-arm64": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/linux-arm64/-/linux-arm64-1.5.2.tgz", + "integrity": "sha512-Ovt9+K35sqzn8H3ZMXGwls4TD/wMJuvRtShHIsmUQREmaxjrDEX7gHckRCrwYJ4XE1H1p6HkLz3wukrAnsfXQw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@pagefind/linux-x64": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/linux-x64/-/linux-x64-1.5.2.tgz", + "integrity": "sha512-V+tFqHKXhQKq/WqPBD67AFy7scn1/aZID00ws4fSDd+1daSi5UHR9VVlRrOUYKxn3VuFQYRD7lYXdZK1WED1YA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@pagefind/windows-arm64": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/windows-arm64/-/windows-arm64-1.5.2.tgz", + "integrity": "sha512-hN9Nh90fNW61nNRCW9ZyQrAj/mD0eRvmJ8NlTUzkbuW8kIzGJUi3cxjFkEcMZ5h/8FsKWD/VcouZl4yo1F7B6g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@pagefind/windows-x64": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/windows-x64/-/windows-x64-1.5.2.tgz", + "integrity": "sha512-Fa2Iyw7kaDRzGMfNYNUXNW2zbL5FQVDgSOcbDHdzBrDEdpqOqg8TcZ68F22ol6NJ9IGzvUdmeyZypLW5dyhqsg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/pluginutils": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.3.0.tgz", + "integrity": "sha512-5EdhGZtnu3V88ces7s53hhfK5KSASnJZv8Lulpc04cWO3REESroJXg73DFsOmgbU2BhwV0E20bu2IDZb3VKW4Q==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "estree-walker": "^2.0.2", + "picomatch": "^4.0.2" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/pluginutils/node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.2.tgz", + "integrity": "sha512-dnlp69efPPg6Uaw2dVqzWRfAWRnYVb1XJ8CyyhIbZeaq4CA5/mLeZ1IEt9QqQxmbdvagjLIm2ZL8BxXv5lH4Yw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.2.tgz", + "integrity": "sha512-OqZTwDRDchGRHHm/hwLOL7uVPB9aUvI0am/eQuWMNyFHf5PSEQmyEeYYheA0EPPKUO/l0uigCp+iaTjoLjVoHg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.2.tgz", + "integrity": "sha512-UwRE7CGpvSVEQS8gUMBe1uADWjNnVgP3Iusyda1nSRwNDCsRjnGc7w6El6WLQsXmZTbLZx9cecegumcitNfpmA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.2.tgz", + "integrity": "sha512-gjEtURKLCC5VXm1I+2i1u9OhxFsKAQJKTVB8WvDAHF+oZlq0GTVFOlTlO1q3AlCTE/DF32c16ESvfgqR7343/g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.2.tgz", + "integrity": "sha512-Bcl6CYDeAgE70cqZaMojOi/eK63h5Me97ZqAQoh77VPjMysA/4ORQBRGo3rRy45x4MzVlU9uZxs8Uwy7ZaKnBw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.2.tgz", + "integrity": "sha512-LU+TPda3mAE2QB0/Hp5VyeKJivpC6+tlOXd1VMoXV/YFMvk/MNk5iXeBfB4MQGRWyOYVJ01625vjkr0Az98OJQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.2.tgz", + "integrity": "sha512-2QxQrM+KQ7DAW4o22j+XZ6RKdxjLD7BOWTP0Bv0tmjdyhXSsr2Ul1oJDQqh9Zf5qOwTuTc7Ek83mOFaKnodPjg==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.2.tgz", + "integrity": "sha512-TbziEu2DVsTEOPif2mKWkMeDMLoYjx95oESa9fkQQK7r/Orta0gnkcDpzwufEcAO2BLBsD7mZkXGFqEdMRRwfw==", + "cpu": [ + "arm" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.2.tgz", + "integrity": "sha512-bO/rVDiDUuM2YfuCUwZ1t1cP+/yqjqz+Xf2VtkdppefuOFS2OSeAfgafaHNkFn0t02hEyXngZkxtGqXcXwO8Rg==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.2.tgz", + "integrity": "sha512-hr26p7e93Rl0Za+JwW7EAnwAvKkehh12BU1Llm9Ykiibg4uIr2rbpxG9WCf56GuvidlTG9KiiQT/TXT1yAWxTA==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.2.tgz", + "integrity": "sha512-pOjB/uSIyDt+ow3k/RcLvUAOGpysT2phDn7TTUB3n75SlIgZzM6NKAqlErPhoFU+npgY3/n+2HYIQVbF70P9/A==", + "cpu": [ + "loong64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.2.tgz", + "integrity": "sha512-2/w+q8jszv9Ww1c+6uJT3OwqhdmGP2/4T17cu8WuwyUuuaCDDJ2ojdyYwZzCxx0GcsZBhzi3HmH+J5pZNXnd+Q==", + "cpu": [ + "loong64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.2.tgz", + "integrity": "sha512-11+aL5vKheYgczxtPVVRhdptAM2H7fcDR5Gw4/bTcteuZBlH4oP9f5s9zYO9aGZvoGeBpqXI/9TZZihZ609wKw==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.2.tgz", + "integrity": "sha512-i16fokAGK46IVZuV8LIIwMdtqhin9hfYkCh8pf8iC3QU3LpwL+1FSFGej+O7l3E/AoknL6Dclh2oTdnRMpTzFQ==", + "cpu": [ + "ppc64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.2.tgz", + "integrity": "sha512-49FkKS6RGQoriDSK/6E2GkAsAuU5kETFCh7pG4yD/ylj9rKhTmO3elsnmBvRD4PgJPds5W2PkhC82aVwmUcJ7A==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.2.tgz", + "integrity": "sha512-mjYNkHPfGpUR00DuM1ZZIgs64Hpf4bWcz9Z41+4Q+pgDx73UwWdAYyf6EG/lRFldmdHHzgrYyge5akFUW0D3mQ==", + "cpu": [ + "riscv64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.2.tgz", + "integrity": "sha512-ALyvJz965BQk8E9Al/JDKKDLH2kfKFLTGMlgkAbbYtZuJt9LU8DW3ZoDMCtQpXAltZxwBHevXz5u+gf0yA0YoA==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.2.tgz", + "integrity": "sha512-UQjrkIdWrKI626Du8lCQ6MJp/6V1LAo2bOK9OTu4mSn8GGXIkPXk/Vsp4bLHCd9Z9Iz2OTEaokUE90VweJgIYQ==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.2.tgz", + "integrity": "sha512-bTsRGj6VlSdn/XD4CGyzMnzaBs9bsRxy79eTqTCBsA8TMIEky7qg48aPkvJvFe1HyzQ5oMZdg7AnVlWQSKLTnw==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.2.tgz", + "integrity": "sha512-6d4Z3534xitaA1FcMWP7mQPq5zGwBmGbhphh2DwaA1aNIXUu3KTOfwrWpbwI4/Gr0uANo7NTtaykFyO2hPuFLg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.2.tgz", + "integrity": "sha512-NetAg5iO2uN7eB8zE5qrZ3CSil+7IJt4WDFLcC75Ymywq1VZVD6qJ6EvNLjZ3rEm6gB7XW5JdT60c6MN35Z85Q==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.2.tgz", + "integrity": "sha512-NCYhOotpgWZ5kdxCZsv6Iudx0wX8980Q/oW4pNFNihpBKsDbEA1zpkfxJGC0yugsUuyDZ7gL37dbzwhR0VI7pQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.2.tgz", + "integrity": "sha512-RXsaOqXxfoUBQoOgvmmijVxJnW2IGB0eoMO7F8FAjaj0UTywUO/luSqimWBJn04WNgUkeNhh7fs7pESXajWmkg==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.2.tgz", + "integrity": "sha512-qdAzEULD+/hzObedtmV6iBpdL5TIbKVztGiK7O3/KYSf+HIzU257+MX1EXJcyIiDbMAqmbwaufcYPvyRryeZtA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.2.tgz", + "integrity": "sha512-Nd/SgG27WoA9e+/TdK74KnHz852TLa94ovOYySo/yMPuTmpckK/jIF2jSwS3g7ELSKXK13/cVdmg1Z/DaCWKxA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@shikijs/core": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@shikijs/core/-/core-4.0.2.tgz", + "integrity": "sha512-hxT0YF4ExEqB8G/qFdtJvpmHXBYJ2lWW7qTHDarVkIudPFE6iCIrqdgWxGn5s+ppkGXI0aEGlibI0PAyzP3zlw==", + "license": "MIT", + "dependencies": { + "@shikijs/primitive": "4.0.2", + "@shikijs/types": "4.0.2", + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.4", + "hast-util-to-html": "^9.0.5" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/engine-javascript": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@shikijs/engine-javascript/-/engine-javascript-4.0.2.tgz", + "integrity": "sha512-7PW0Nm49DcoUIQEXlJhNNBHyoGMjalRETTCcjMqEaMoJRLljy1Bi/EGV3/qLBgLKQejdspiiYuHGQW6dX94Nag==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "4.0.2", + "@shikijs/vscode-textmate": "^10.0.2", + "oniguruma-to-es": "^4.3.4" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/engine-oniguruma": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@shikijs/engine-oniguruma/-/engine-oniguruma-4.0.2.tgz", + "integrity": "sha512-UpCB9Y2sUKlS9z8juFSKz7ZtysmeXCgnRF0dlhXBkmQnek7lAToPte8DkxmEYGNTMii72zU/lyXiCB6StuZeJg==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "4.0.2", + "@shikijs/vscode-textmate": "^10.0.2" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/langs": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@shikijs/langs/-/langs-4.0.2.tgz", + "integrity": "sha512-KaXby5dvoeuZzN0rYQiPMjFoUrz4hgwIE+D6Du9owcHcl6/g16/yT5BQxSW5cGt2MZBz6Hl0YuRqf12omRfUUg==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "4.0.2" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/primitive": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@shikijs/primitive/-/primitive-4.0.2.tgz", + "integrity": "sha512-M6UMPrSa3fN5ayeJwFVl9qWofl273wtK1VG8ySDZ1mQBfhCpdd8nEx7nPZ/tk7k+TYcpqBZzj/AnwxT9lO+HJw==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "4.0.2", + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.4" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/themes": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@shikijs/themes/-/themes-4.0.2.tgz", + "integrity": "sha512-mjCafwt8lJJaVSsQvNVrJumbnnj1RI8jbUKrPKgE6E3OvQKxnuRoBaYC51H4IGHePsGN/QtALglWBU7DoKDFnA==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "4.0.2" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/types": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-4.0.2.tgz", + "integrity": "sha512-qzbeRooUTPnLE+sHD/Z8DStmaDgnbbc/pMrU203950aRqjX/6AFHeDYT+j00y2lPdz0ywJKx7o/7qnqTivtlXg==", + "license": "MIT", + "dependencies": { + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.4" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/vscode-textmate": { + "version": "10.0.2", + "resolved": "https://registry.npmjs.org/@shikijs/vscode-textmate/-/vscode-textmate-10.0.2.tgz", + "integrity": "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==", + "license": "MIT" + }, + "node_modules/@types/debug": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "license": "MIT" + }, + "node_modules/@types/estree-jsx": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", + "integrity": "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==", + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/js-yaml": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", + "license": "MIT" + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/mdx": { + "version": "2.0.13", + "resolved": "https://registry.npmjs.org/@types/mdx/-/mdx-2.0.13.tgz", + "integrity": "sha512-+OWZQfAYyio6YkJb3HLxDrvnx6SWWDbC0zVPfBRzUk0/nqoDyf6dNxQi3eArPe8rJ473nobTMQ/8Zk+LxJ+Yuw==", + "license": "MIT" + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, + "node_modules/@types/nlcst": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/nlcst/-/nlcst-2.0.3.tgz", + "integrity": "sha512-vSYNSDe6Ix3q+6Z7ri9lyWqgGhJTmzRjZRqyq15N0Z/1/UnVsno9G/N40NBijoYx2seFDIl0+B2mgAb9mezUCA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/node": { + "version": "24.12.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.2.tgz", + "integrity": "sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/@types/sax": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz", + "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, + "node_modules/@ungap/structured-clone": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", + "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", + "license": "ISC" + }, + "node_modules/@volar/kit": { + "version": "2.4.28", + "resolved": "https://registry.npmjs.org/@volar/kit/-/kit-2.4.28.tgz", + "integrity": "sha512-cKX4vK9dtZvDRaAzeoUdaAJEew6IdxHNCRrdp5Kvcl6zZOqb6jTOfk3kXkIkG3T7oTFXguEMt5+9ptyqYR84Pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@volar/language-service": "2.4.28", + "@volar/typescript": "2.4.28", + "typesafe-path": "^0.2.2", + "vscode-languageserver-textdocument": "^1.0.11", + "vscode-uri": "^3.0.8" + }, + "peerDependencies": { + "typescript": "*" + } + }, + "node_modules/@volar/language-core": { + "version": "2.4.28", + "resolved": "https://registry.npmjs.org/@volar/language-core/-/language-core-2.4.28.tgz", + "integrity": "sha512-w4qhIJ8ZSitgLAkVay6AbcnC7gP3glYM3fYwKV3srj8m494E3xtrCv6E+bWviiK/8hs6e6t1ij1s2Endql7vzQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@volar/source-map": "2.4.28" + } + }, + "node_modules/@volar/language-server": { + "version": "2.4.28", + "resolved": "https://registry.npmjs.org/@volar/language-server/-/language-server-2.4.28.tgz", + "integrity": "sha512-NqcLnE5gERKuS4PUFwlhMxf6vqYo7hXtbMFbViXcbVkbZ905AIVWhnSo0ZNBC2V127H1/2zP7RvVOVnyITFfBw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@volar/language-core": "2.4.28", + "@volar/language-service": "2.4.28", + "@volar/typescript": "2.4.28", + "path-browserify": "^1.0.1", + "request-light": "^0.7.0", + "vscode-languageserver": "^9.0.1", + "vscode-languageserver-protocol": "^3.17.5", + "vscode-languageserver-textdocument": "^1.0.11", + "vscode-uri": "^3.0.8" + } + }, + "node_modules/@volar/language-service": { + "version": "2.4.28", + "resolved": "https://registry.npmjs.org/@volar/language-service/-/language-service-2.4.28.tgz", + "integrity": "sha512-Rh/wYCZJrI5vCwMk9xyw/Z+MsWxlJY1rmMZPsxUoJKfzIRjS/NF1NmnuEcrMbEVGja00aVpCsInJfixQTMdvLw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@volar/language-core": "2.4.28", + "vscode-languageserver-protocol": "^3.17.5", + "vscode-languageserver-textdocument": "^1.0.11", + "vscode-uri": "^3.0.8" + } + }, + "node_modules/@volar/source-map": { + "version": "2.4.28", + "resolved": "https://registry.npmjs.org/@volar/source-map/-/source-map-2.4.28.tgz", + "integrity": "sha512-yX2BDBqJkRXfKw8my8VarTyjv48QwxdJtvRgUpNE5erCsgEUdI2DsLbpa+rOQVAJYshY99szEcRDmyHbF10ggQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@volar/typescript": { + "version": "2.4.28", + "resolved": "https://registry.npmjs.org/@volar/typescript/-/typescript-2.4.28.tgz", + "integrity": "sha512-Ja6yvWrbis2QtN4ClAKreeUZPVYMARDYZl9LMEv1iQ1QdepB6wn0jTRxA9MftYmYa4DQ4k/DaSZpFPUfxl8giw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@volar/language-core": "2.4.28", + "path-browserify": "^1.0.1", + "vscode-uri": "^3.0.8" + } + }, + "node_modules/@vscode/emmet-helper": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@vscode/emmet-helper/-/emmet-helper-2.11.0.tgz", + "integrity": "sha512-QLxjQR3imPZPQltfbWRnHU6JecWTF1QSWhx3GAKQpslx7y3Dp6sIIXhKjiUJ/BR9FX8PVthjr9PD6pNwOJfAzw==", + "dev": true, + "license": "MIT", + "dependencies": { + "emmet": "^2.4.3", + "jsonc-parser": "^2.3.0", + "vscode-languageserver-textdocument": "^1.0.1", + "vscode-languageserver-types": "^3.15.1", + "vscode-uri": "^3.0.8" + } + }, + "node_modules/@vscode/l10n": { + "version": "0.0.18", + "resolved": "https://registry.npmjs.org/@vscode/l10n/-/l10n-0.0.18.tgz", + "integrity": "sha512-KYSIHVmslkaCDyw013pphY+d7x1qV8IZupYfeIfzNA+nsaWHbn5uPuQRvdRFsa9zFzGeudPuoGoZ1Op4jrJXIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/acorn": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-draft-04": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/ajv-draft-04/-/ajv-draft-04-1.0.0.tgz", + "integrity": "sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "ajv": "^8.5.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/anymatch/node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "license": "MIT" + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/aria-query": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.2.tgz", + "integrity": "sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==", + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/array-iterate": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/array-iterate/-/array-iterate-2.0.1.tgz", + "integrity": "sha512-I1jXZMjAgCMmxT4qxXfPXa6SthSoE8h6gkSI9BGGNv8mP8G/v0blc+qFnZu6K42vTOiuME596QaLO0TP3Lk0xg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/astring": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/astring/-/astring-1.9.0.tgz", + "integrity": "sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==", + "license": "MIT", + "bin": { + "astring": "bin/astring" + } + }, + "node_modules/astro": { + "version": "6.1.9", + "resolved": "https://registry.npmjs.org/astro/-/astro-6.1.9.tgz", + "integrity": "sha512-NsAHzMzpznB281g2aM5qnBt2QjfH6ttKiZ3hSZw52If8JJ+62kbnBKbyKhR2glQcJLl7Jfe4GSl0DihFZ36rRQ==", + "license": "MIT", + "dependencies": { + "@astrojs/compiler": "^3.0.1", + "@astrojs/internal-helpers": "0.9.0", + "@astrojs/markdown-remark": "7.1.1", + "@astrojs/telemetry": "3.3.1", + "@capsizecss/unpack": "^4.0.0", + "@clack/prompts": "^1.1.0", + "@oslojs/encoding": "^1.1.0", + "@rollup/pluginutils": "^5.3.0", + "aria-query": "^5.3.2", + "axobject-query": "^4.1.0", + "ci-info": "^4.4.0", + "clsx": "^2.1.1", + "common-ancestor-path": "^2.0.0", + "cookie": "^1.1.1", + "devalue": "^5.6.3", + "diff": "^8.0.3", + "dset": "^3.1.4", + "es-module-lexer": "^2.0.0", + "esbuild": "^0.27.3", + "flattie": "^1.1.1", + "fontace": "~0.4.1", + "github-slugger": "^2.0.0", + "html-escaper": "3.0.3", + "http-cache-semantics": "^4.2.0", + "js-yaml": "^4.1.1", + "magic-string": "^0.30.21", + "magicast": "^0.5.2", + "mrmime": "^2.0.1", + "neotraverse": "^0.6.18", + "obug": "^2.1.1", + "p-limit": "^7.3.0", + "p-queue": "^9.1.0", + "package-manager-detector": "^1.6.0", + "piccolore": "^0.1.3", + "picomatch": "^4.0.4", + "rehype": "^13.0.2", + "semver": "^7.7.4", + "shiki": "^4.0.2", + "smol-toml": "^1.6.0", + "svgo": "^4.0.1", + "tinyclip": "^0.1.12", + "tinyexec": "^1.0.4", + "tinyglobby": "^0.2.15", + "tsconfck": "^3.1.6", + "ultrahtml": "^1.6.0", + "unifont": "~0.7.4", + "unist-util-visit": "^5.1.0", + "unstorage": "^1.17.5", + "vfile": "^6.0.3", + "vite": "^7.3.2", + "vitefu": "^1.1.2", + "xxhash-wasm": "^1.1.0", + "yargs-parser": "^22.0.0", + "zod": "^4.3.6" + }, + "bin": { + "astro": "bin/astro.mjs" + }, + "engines": { + "node": ">=22.12.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/astrodotbuild" + }, + "optionalDependencies": { + "sharp": "^0.34.0" + } + }, + "node_modules/astro-expressive-code": { + "version": "0.41.7", + "resolved": "https://registry.npmjs.org/astro-expressive-code/-/astro-expressive-code-0.41.7.tgz", + "integrity": "sha512-hUpogGc6DdAd+I7pPXsctyYPRBJDK7Q7d06s4cyP0Vz3OcbziP3FNzN0jZci1BpCvLn9675DvS7B9ctKKX64JQ==", + "license": "MIT", + "dependencies": { + "rehype-expressive-code": "^0.41.7" + }, + "peerDependencies": { + "astro": "^4.0.0-beta || ^5.0.0-beta || ^3.3.0 || ^6.0.0-beta" + } + }, + "node_modules/axobject-query": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", + "integrity": "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==", + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/bcp-47": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/bcp-47/-/bcp-47-2.1.0.tgz", + "integrity": "sha512-9IIS3UPrvIa1Ej+lVDdDwO7zLehjqsaByECw0bu2RRGP73jALm6FYbzI5gWbgHLvNdkvfXB5YrSbocZdOS0c0w==", + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/bcp-47-match": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/bcp-47-match/-/bcp-47-match-2.0.3.tgz", + "integrity": "sha512-JtTezzbAibu8G0R9op9zb3vcWZd9JF6M0xOYGPn0fNCd7wOpRB1mU2mH9T8gaBGbAAyIIVgB2G7xG0GP98zMAQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/chokidar": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", + "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", + "license": "MIT", + "dependencies": { + "readdirp": "^5.0.0" + }, + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/ci-info": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.4.0.tgz", + "integrity": "sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/collapse-white-space": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/collapse-white-space/-/collapse-white-space-2.1.0.tgz", + "integrity": "sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/commander": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", + "integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==", + "license": "MIT", + "engines": { + "node": ">=16" + } + }, + "node_modules/common-ancestor-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/common-ancestor-path/-/common-ancestor-path-2.0.0.tgz", + "integrity": "sha512-dnN3ibLeoRf2HNC+OlCiNc5d2zxbLJXOtiZUudNFSXZrNSydxcCsSpRzXwfu7BBWCIfHPw+xTayeBvJCP/D8Ng==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">= 18" + } + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cookie-es": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/cookie-es/-/cookie-es-1.2.3.tgz", + "integrity": "sha512-lXVyvUvrNXblMqzIRrxHb57UUVmqsSWlxqt3XIjCkUP0wDAf6uicO6KMbEgYrMNtEvWgWHwe42CKxPu9MYAnWw==", + "license": "MIT" + }, + "node_modules/crossws": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/crossws/-/crossws-0.3.5.tgz", + "integrity": "sha512-ojKiDvcmByhwa8YYqbQI/hg7MEU0NC03+pSdEq4ZUnZR9xXpwk7E43SMNGkn+JxJGPFtNvQ48+vV2p+P1ml5PA==", + "license": "MIT", + "dependencies": { + "uncrypto": "^0.1.3" + } + }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-selector-parser": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/css-selector-parser/-/css-selector-parser-3.3.0.tgz", + "integrity": "sha512-Y2asgMGFqJKF4fq4xHDSlFYIkeVfRsm69lQC1q9kbEsH5XtnINTMrweLkjYMeaUgiXBy/uvKeO/a1JHTNnmB2g==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/mdevils" + }, + { + "type": "patreon", + "url": "https://patreon.com/mdevils" + } + ], + "license": "MIT" + }, + "node_modules/css-tree": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", + "integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==", + "license": "MIT", + "dependencies": { + "mdn-data": "2.27.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "license": "MIT", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/csso": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/csso/-/csso-5.0.5.tgz", + "integrity": "sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==", + "license": "MIT", + "dependencies": { + "css-tree": "~2.2.0" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/csso/node_modules/css-tree": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.2.1.tgz", + "integrity": "sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==", + "license": "MIT", + "dependencies": { + "mdn-data": "2.0.28", + "source-map-js": "^1.0.1" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/csso/node_modules/mdn-data": { + "version": "2.0.28", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.28.tgz", + "integrity": "sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==", + "license": "CC0-1.0" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/defu": { + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.7.tgz", + "integrity": "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==", + "license": "MIT" + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/destr": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/destr/-/destr-2.0.5.tgz", + "integrity": "sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA==", + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/devalue": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.7.1.tgz", + "integrity": "sha512-MUbZ586EgQqdRnC4yDrlod3BEdyvE4TapGYHMW2CiaW+KkkFmWEFqBUaLltEZCGi0iFXCEjRF0OjF0DV2QHjOA==", + "license": "MIT" + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/diff": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz", + "integrity": "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/direction": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/direction/-/direction-2.0.1.tgz", + "integrity": "sha512-9S6m9Sukh1cZNknO1CWAr2QAWsbKLafQiyM5gZ7VgXHeuaoUwffKN4q6NC4A/Mf9iiPlOXQEKW/Mv/mh9/3YFA==", + "license": "MIT", + "bin": { + "direction": "cli.js" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dlv": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", + "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", + "license": "MIT" + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/dom-serializer/node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/dset": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.4.tgz", + "integrity": "sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/emmet": { + "version": "2.4.11", + "resolved": "https://registry.npmjs.org/emmet/-/emmet-2.4.11.tgz", + "integrity": "sha512-23QPJB3moh/U9sT4rQzGgeyyGIrcM+GH5uVYg2C6wZIxAIJq7Ng3QLT79tl8FUwDXhyq9SusfknOrofAKqvgyQ==", + "dev": true, + "license": "MIT", + "workspaces": [ + "./packages/scanner", + "./packages/abbreviation", + "./packages/css-abbreviation", + "./" + ], + "dependencies": { + "@emmetio/abbreviation": "^2.3.3", + "@emmetio/css-abbreviation": "^2.1.8" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-module-lexer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.0.0.tgz", + "integrity": "sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==", + "license": "MIT" + }, + "node_modules/esast-util-from-estree": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/esast-util-from-estree/-/esast-util-from-estree-2.0.0.tgz", + "integrity": "sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "devlop": "^1.0.0", + "estree-util-visit": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/esast-util-from-js": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/esast-util-from-js/-/esast-util-from-js-2.0.1.tgz", + "integrity": "sha512-8Ja+rNJ0Lt56Pcf3TAmpBZjmx8ZcK5Ts4cAzIOjsjevg9oSXJnl6SUQ2EevU8tv3h6ZLWmoKL5H4fgWvdvfETw==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "acorn": "^8.0.0", + "esast-util-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/esbuild": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/estree-util-attach-comments": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-attach-comments/-/estree-util-attach-comments-3.0.0.tgz", + "integrity": "sha512-cKUwm/HUcTDsYh/9FgnuFqpfquUbwIqwKM26BVCGDPVgvaCl/nDCCjUfiLlx6lsEZ3Z4RFxNbOQ60pkaEwFxGw==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-build-jsx": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/estree-util-build-jsx/-/estree-util-build-jsx-3.0.1.tgz", + "integrity": "sha512-8U5eiL6BTrPxp/CHbs2yMgP8ftMhR5ww1eIKoWRMlqvltHF8fZn5LRDvTKuxD3DUn+shRbLGqXemcP51oFCsGQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "estree-walker": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-is-identifier-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz", + "integrity": "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-scope": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/estree-util-scope/-/estree-util-scope-1.0.0.tgz", + "integrity": "sha512-2CAASclonf+JFWBNJPndcOpA8EMJwa0Q8LUFJEKqXLW6+qBvbFZuF5gItbQOs/umBUkjviCSDCbBwU2cXbmrhQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-to-js": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/estree-util-to-js/-/estree-util-to-js-2.0.0.tgz", + "integrity": "sha512-WDF+xj5rRWmD5tj6bIqRi6CkLIXbbNQUcxQHzGysQzvHmdYG2G7p/Tf0J0gpxGgkeMZNTIjT/AoSvC9Xehcgdg==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "astring": "^1.8.0", + "source-map": "^0.7.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-visit": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/estree-util-visit/-/estree-util-visit-2.0.0.tgz", + "integrity": "sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/expressive-code": { + "version": "0.41.7", + "resolved": "https://registry.npmjs.org/expressive-code/-/expressive-code-0.41.7.tgz", + "integrity": "sha512-2wZjC8OQ3TaVEMcBtYY4Va3lo6J+Ai9jf3d4dbhURMJcU4Pbqe6EcHe424MIZI0VHUA1bR6xdpoHYi3yxokWqA==", + "license": "MIT", + "dependencies": { + "@expressive-code/core": "^0.41.7", + "@expressive-code/plugin-frames": "^0.41.7", + "@expressive-code/plugin-shiki": "^0.41.7", + "@expressive-code/plugin-text-markers": "^0.41.7" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-string-truncated-width": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/fast-string-truncated-width/-/fast-string-truncated-width-1.2.1.tgz", + "integrity": "sha512-Q9acT/+Uu3GwGj+5w/zsGuQjh9O1TyywhIwAxHudtWrgF09nHOPrvTLhQevPbttcxjr/SNN7mJmfOw/B1bXgow==", + "license": "MIT" + }, + "node_modules/fast-string-width": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fast-string-width/-/fast-string-width-1.1.0.tgz", + "integrity": "sha512-O3fwIVIH5gKB38QNbdg+3760ZmGz0SZMgvwJbA1b2TGXceKE6A2cOlfogh1iw8lr049zPyd7YADHy+B7U4W9bQ==", + "license": "MIT", + "dependencies": { + "fast-string-truncated-width": "^1.2.0" + } + }, + "node_modules/fast-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", + "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fast-wrap-ansi": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.1.6.tgz", + "integrity": "sha512-HlUwET7a5gqjURj70D5jl7aC3Zmy4weA1SHUfM0JFI0Ptq987NH2TwbBFLoERhfwk+E+eaq4EK3jXoT+R3yp3w==", + "license": "MIT", + "dependencies": { + "fast-string-width": "^1.1.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/flattie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/flattie/-/flattie-1.1.1.tgz", + "integrity": "sha512-9UbaD6XdAL97+k/n+N7JwX46K/M6Zc6KcFYskrYL8wbBV/Uyk0CTAMY0VT+qiK5PM7AIc9aTWYtq65U7T+aCNQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/fontace": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/fontace/-/fontace-0.4.1.tgz", + "integrity": "sha512-lDMvbAzSnHmbYMTEld5qdtvNH2/pWpICOqpean9IgC7vUbUJc3k+k5Dokp85CegamqQpFbXf0rAVkbzpyTA8aw==", + "license": "MIT", + "dependencies": { + "fontkitten": "^1.0.2" + } + }, + "node_modules/fontkitten": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/fontkitten/-/fontkitten-1.0.3.tgz", + "integrity": "sha512-Wp1zXWPVUPBmfoa3Cqc9ctaKuzKAV6uLstRqlR56kSjplf5uAce+qeyYym7F+PHbGTk+tCEdkCW6RD7DX/gBZw==", + "license": "MIT", + "dependencies": { + "tiny-inflate": "^1.0.3" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/github-slugger": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-2.0.0.tgz", + "integrity": "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==", + "license": "ISC" + }, + "node_modules/h3": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/h3/-/h3-1.15.11.tgz", + "integrity": "sha512-L3THSe2MPeBwgIZVSH5zLdBBU90TOxarvhK9d04IDY2AmVS8j2Jz2LIWtwsGOU3lu2I5jCN7FNvVfY2+XyF+mg==", + "license": "MIT", + "dependencies": { + "cookie-es": "^1.2.3", + "crossws": "^0.3.5", + "defu": "^6.1.6", + "destr": "^2.0.5", + "iron-webcrypto": "^1.2.1", + "node-mock-http": "^1.0.4", + "radix3": "^1.1.2", + "ufo": "^1.6.3", + "uncrypto": "^0.1.3" + } + }, + "node_modules/hast-util-embedded": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-embedded/-/hast-util-embedded-3.0.0.tgz", + "integrity": "sha512-naH8sld4Pe2ep03qqULEtvYr7EjrLK2QHY8KJR6RJkTUjPGObe1vnx585uzem2hGra+s1q08DZZpfgDVYRbaXA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-is-element": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-format": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/hast-util-format/-/hast-util-format-1.1.0.tgz", + "integrity": "sha512-yY1UDz6bC9rDvCWHpx12aIBGRG7krurX0p0Fm6pT547LwDIZZiNr8a+IHDogorAdreULSEzP82Nlv5SZkHZcjA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-embedded": "^3.0.0", + "hast-util-minify-whitespace": "^1.0.0", + "hast-util-phrasing": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "html-whitespace-sensitive-tag-names": "^3.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-from-html": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hast-util-from-html/-/hast-util-from-html-2.0.3.tgz", + "integrity": "sha512-CUSRHXyKjzHov8yKsQjGOElXy/3EKpyX56ELnkHH34vDVw1N1XSQ1ZcAvTyAPtGqLTuKP/uxM+aLkSPqF/EtMw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "devlop": "^1.1.0", + "hast-util-from-parse5": "^8.0.0", + "parse5": "^7.0.0", + "vfile": "^6.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-from-parse5": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-8.0.3.tgz", + "integrity": "sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "devlop": "^1.0.0", + "hastscript": "^9.0.0", + "property-information": "^7.0.0", + "vfile": "^6.0.0", + "vfile-location": "^5.0.0", + "web-namespaces": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-has-property": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-has-property/-/hast-util-has-property-3.0.0.tgz", + "integrity": "sha512-MNilsvEKLFpV604hwfhVStK0usFY/QmM5zX16bo7EjnAEGofr5YyI37kzopBlZJkHD4t887i+q/C8/tr5Q94cA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-is-body-ok-link": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/hast-util-is-body-ok-link/-/hast-util-is-body-ok-link-3.0.1.tgz", + "integrity": "sha512-0qpnzOBLztXHbHQenVB8uNuxTnm/QBFUOmdOSsEn7GnBtyY07+ENTWVFBAnXd/zEgd9/SUG3lRY7hSIBWRgGpQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-is-element": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-3.0.0.tgz", + "integrity": "sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-minify-whitespace": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hast-util-minify-whitespace/-/hast-util-minify-whitespace-1.0.1.tgz", + "integrity": "sha512-L96fPOVpnclQE0xzdWb/D12VT5FabA7SnZOUMtL1DbXmYiHJMXZvFkIZfiMmTCNJHUeO2K9UYNXoVyfz+QHuOw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-embedded": "^3.0.0", + "hast-util-is-element": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-parse-selector": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz", + "integrity": "sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-phrasing": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/hast-util-phrasing/-/hast-util-phrasing-3.0.1.tgz", + "integrity": "sha512-6h60VfI3uBQUxHqTyMymMZnEbNl1XmEGtOxxKYL7stY2o601COo62AWAYBQR9lZbYXYSBoxag8UpPRXK+9fqSQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-embedded": "^3.0.0", + "hast-util-has-property": "^3.0.0", + "hast-util-is-body-ok-link": "^3.0.0", + "hast-util-is-element": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-raw": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/hast-util-raw/-/hast-util-raw-9.1.0.tgz", + "integrity": "sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "@ungap/structured-clone": "^1.0.0", + "hast-util-from-parse5": "^8.0.0", + "hast-util-to-parse5": "^8.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "parse5": "^7.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-select": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/hast-util-select/-/hast-util-select-6.0.4.tgz", + "integrity": "sha512-RqGS1ZgI0MwxLaKLDxjprynNzINEkRHY2i8ln4DDjgv9ZhcYVIHN9rlpiYsqtFwrgpYU361SyWDQcGNIBVu3lw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "bcp-47-match": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "css-selector-parser": "^3.0.0", + "devlop": "^1.0.0", + "direction": "^2.0.0", + "hast-util-has-property": "^3.0.0", + "hast-util-to-string": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "nth-check": "^2.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-estree": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/hast-util-to-estree/-/hast-util-to-estree-3.1.3.tgz", + "integrity": "sha512-48+B/rJWAp0jamNbAAf9M7Uf//UVqAoMmgXhBdxTDJLGKY+LRnZ99qcG+Qjl5HfMpYNzS5v4EAwVEF34LeAj7w==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-attach-comments": "^3.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-js": "^1.0.0", + "unist-util-position": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-html": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.5.tgz", + "integrity": "sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-whitespace": "^3.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "stringify-entities": "^4.0.0", + "zwitch": "^2.0.4" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-jsx-runtime": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.6.tgz", + "integrity": "sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-js": "^1.0.0", + "unist-util-position": "^5.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-parse5": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/hast-util-to-parse5/-/hast-util-to-parse5-8.0.1.tgz", + "integrity": "sha512-MlWT6Pjt4CG9lFCjiz4BH7l9wmrMkfkJYCxFwKQic8+RTZgWPuWxwAfjJElsXkex7DJjfSJsQIt931ilUgmwdA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-string": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/hast-util-to-string/-/hast-util-to-string-3.0.1.tgz", + "integrity": "sha512-XelQVTDWvqcl3axRfI0xSeoVKzyIFPwsAGSLIsKdJKQMXDYJS4WYrBNF/8J7RdhIcFI2BOHgAifggsvsxp/3+A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-text": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/hast-util-to-text/-/hast-util-to-text-4.0.2.tgz", + "integrity": "sha512-KK6y/BN8lbaq654j7JgBydev7wuNMcID54lkRav1P0CaE1e47P72AWWPiGKXTJU271ooYzcvTAn/Zt0REnvc7A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "hast-util-is-element": "^3.0.0", + "unist-util-find-after": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-whitespace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", + "integrity": "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hastscript": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-9.0.1.tgz", + "integrity": "sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-parse-selector": "^4.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/html-escaper": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz", + "integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==", + "license": "MIT" + }, + "node_modules/html-void-elements": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-3.0.0.tgz", + "integrity": "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/html-whitespace-sensitive-tag-names": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/html-whitespace-sensitive-tag-names/-/html-whitespace-sensitive-tag-names-3.0.1.tgz", + "integrity": "sha512-q+310vW8zmymYHALr1da4HyXUQ0zgiIwIicEfotYPWGN0OJVEN/58IJ3A4GBYcEq3LGAZqKb+ugvP0GNB9CEAA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/http-cache-semantics": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", + "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", + "license": "BSD-2-Clause" + }, + "node_modules/i18next": { + "version": "23.16.8", + "resolved": "https://registry.npmjs.org/i18next/-/i18next-23.16.8.tgz", + "integrity": "sha512-06r/TitrM88Mg5FdUXAKL96dJMzgqLE5dv3ryBAra4KCwD9mJ4ndOTS95ZuymIGoE+2hzfdaMak2X11/es7ZWg==", + "funding": [ + { + "type": "individual", + "url": "https://locize.com" + }, + { + "type": "individual", + "url": "https://locize.com/i18next.html" + }, + { + "type": "individual", + "url": "https://www.i18next.com/how-to/faq#i18next-is-awesome.-how-can-i-support-the-project" + } + ], + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.23.2" + } + }, + "node_modules/inline-style-parser": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.2.7.tgz", + "integrity": "sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA==", + "license": "MIT" + }, + "node_modules/iron-webcrypto": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/iron-webcrypto/-/iron-webcrypto-1.2.1.tgz", + "integrity": "sha512-feOM6FaSr6rEABp/eDfVseKyTMDt+KGpeB35SkVn9Tyn0CqvVsY3EwI0v5i8nMHyJnzCIQf7nsy3p41TPkJZhg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/brc-dd" + } + }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-docker": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-4.0.0.tgz", + "integrity": "sha512-LHE+wROyG/Y/0ZnbktRCoTix2c1RhgWaZraMZ8o1Q7zCh0VSrICJQO5oqIIISrcSBtrXv0o233w1IYwsWCjTzA==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-inside-container": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", + "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", + "license": "MIT", + "dependencies": { + "is-docker": "^3.0.0" + }, + "bin": { + "is-inside-container": "cli.js" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-inside-container/node_modules/is-docker": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", + "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-wsl": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.1.tgz", + "integrity": "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==", + "license": "MIT", + "dependencies": { + "is-inside-container": "^1.0.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/jsonc-parser": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-2.3.1.tgz", + "integrity": "sha512-H8jvkz1O50L3dMZCsLqiuB2tA7muqbSg1AtGEkN0leAqGjsUzDJir3Zwr02BhqdcITPg3ei3mZ+HjMocAknhhg==", + "dev": true, + "license": "MIT" + }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/klona": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/klona/-/klona-2.0.6.tgz", + "integrity": "sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/lru-cache": { + "version": "11.3.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.5.tgz", + "integrity": "sha512-NxVFwLAnrd9i7KUBxC4DrUhmgjzOs+1Qm50D3oF1/oL+r1NpZ4gA7xvG0/zJ8evR7zIKn4vLf7qTNduWFtCrRw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/magicast": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.2.tgz", + "integrity": "sha512-E3ZJh4J3S9KfwdjZhe2afj6R9lGIN5Pher1pF39UGrXRqq/VDaGVIGN13BjHd2u8B61hArAGOnso7nBOouW3TQ==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.0", + "@babel/types": "^7.29.0", + "source-map-js": "^1.2.1" + } + }, + "node_modules/markdown-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/markdown-extensions/-/markdown-extensions-2.0.0.tgz", + "integrity": "sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/markdown-table": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", + "integrity": "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/mdast-util-definitions": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-definitions/-/mdast-util-definitions-6.0.0.tgz", + "integrity": "sha512-scTllyX6pnYNZH/AIp/0ePz6s4cZtARxImwoPJ7kS42n+MnVsI4XbnG6d4ibehRIldYMWM2LD7ImQblVhUejVQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-directive": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-directive/-/mdast-util-directive-3.1.0.tgz", + "integrity": "sha512-I3fNFt+DHmpWCYAT7quoM6lHf9wuqtI+oCOfvILnoicNIqjh5E3dEJWiXuYME2gNe8vl1iMQwyUHa7bgFmak6Q==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-find-and-replace": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.2.tgz", + "integrity": "sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz", + "integrity": "sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.1.0.tgz", + "integrity": "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ==", + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-gfm-autolink-literal": "^2.0.0", + "mdast-util-gfm-footnote": "^2.0.0", + "mdast-util-gfm-strikethrough": "^2.0.0", + "mdast-util-gfm-table": "^2.0.0", + "mdast-util-gfm-task-list-item": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.0.1.tgz", + "integrity": "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "ccount": "^2.0.0", + "devlop": "^1.0.0", + "mdast-util-find-and-replace": "^3.0.0", + "micromark-util-character": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-strikethrough": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.0.tgz", + "integrity": "sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-table": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.0.tgz", + "integrity": "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-task-list-item": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.0.tgz", + "integrity": "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx/-/mdast-util-mdx-3.0.0.tgz", + "integrity": "sha512-JfbYLAW7XnYTTbUsmpu0kdBUVe+yKVJZBItEjwyYJiDJuZ9w4eeaqks4HQO+R7objWgS2ymV60GYpI14Ug554w==", + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-expression": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz", + "integrity": "sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-jsx": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-3.2.0.tgz", + "integrity": "sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-stringify-position": "^4.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdxjs-esm": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-2.0.1.tgz", + "integrity": "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-phrasing": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", + "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-hast": { + "version": "13.2.1", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.1.tgz", + "integrity": "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@ungap/structured-clone": "^1.0.0", + "devlop": "^1.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "trim-lines": "^3.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.1.2.tgz", + "integrity": "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdn-data": { + "version": "2.27.1", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", + "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", + "license": "CC0-1.0" + }, + "node_modules/micromark": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", + "integrity": "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.3.tgz", + "integrity": "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-directive": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/micromark-extension-directive/-/micromark-extension-directive-3.0.2.tgz", + "integrity": "sha512-wjcXHgk+PPdmvR58Le9d7zQYWy+vKEU9Se44p2CrCDPiLr2FMyiT4Fyb5UFKFC66wGB3kPlgD7q3TnoqPS7SZA==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "parse-entities": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", + "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", + "license": "MIT", + "dependencies": { + "micromark-extension-gfm-autolink-literal": "^2.0.0", + "micromark-extension-gfm-footnote": "^2.0.0", + "micromark-extension-gfm-strikethrough": "^2.0.0", + "micromark-extension-gfm-table": "^2.0.0", + "micromark-extension-gfm-tagfilter": "^2.0.0", + "micromark-extension-gfm-task-list-item": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-strikethrough": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", + "integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.1.tgz", + "integrity": "sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", + "integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==", + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", + "integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdx-expression": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-expression/-/micromark-extension-mdx-expression-3.0.1.tgz", + "integrity": "sha512-dD/ADLJ1AeMvSAKBwO22zG22N4ybhe7kFIZ3LsDI0GlsNr2A3KYxb0LdC1u5rj4Nw+CHKY0RVdnHX8vj8ejm4Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-mdx-expression": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-mdx-jsx": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-jsx/-/micromark-extension-mdx-jsx-3.0.2.tgz", + "integrity": "sha512-e5+q1DjMh62LZAJOnDraSSbDMvGJ8x3cbjygy2qFEi7HCeUT4BDKCvMozPozcD6WmOt6sVvYDNBKhFSz3kjOVQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "micromark-factory-mdx-expression": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdx-md": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-md/-/micromark-extension-mdx-md-2.0.0.tgz", + "integrity": "sha512-EpAiszsB3blw4Rpba7xTOUptcFeBFi+6PY8VnJ2hhimH+vCQDirWgsMpz7w1XcZE7LVrSAUGb9VJpG9ghlYvYQ==", + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdxjs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs/-/micromark-extension-mdxjs-3.0.0.tgz", + "integrity": "sha512-A873fJfhnJ2siZyUrJ31l34Uqwy4xIFmvPY1oj+Ean5PHcPBYzEsvqvWGaWcfEIr11O5Dlw3p2y0tZWpKHDejQ==", + "license": "MIT", + "dependencies": { + "acorn": "^8.0.0", + "acorn-jsx": "^5.0.0", + "micromark-extension-mdx-expression": "^3.0.0", + "micromark-extension-mdx-jsx": "^3.0.0", + "micromark-extension-mdx-md": "^2.0.0", + "micromark-extension-mdxjs-esm": "^3.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdxjs-esm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs-esm/-/micromark-extension-mdxjs-esm-3.0.0.tgz", + "integrity": "sha512-DJFl4ZqkErRpq/dAPyeWp15tGrcrrJho1hKK5uBS70BCtfrIFg81sqcTVu3Ta+KD1Tk5vAtBNElWxtAa+m8K9A==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-mdx-expression": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-factory-mdx-expression/-/micromark-factory-mdx-expression-2.0.3.tgz", + "integrity": "sha512-kQnEtA3vzucU2BkrIa8/VaSAsP+EJ3CKOvhMuJgOEGg9KDC6OAY6nSnNDVRiVNRqj7Y4SlSzcStaH/5jge8JdQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", + "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-events-to-acorn": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-util-events-to-acorn/-/micromark-util-events-to-acorn-2.0.3.tgz", + "integrity": "sha512-jmsiEIiZ1n7X1Rr5k8wVExBQCg5jy4UXVADItHmNk1zkwEVhBuIUKRu3fqv+hs4nxLISi2DQGlqIOGiFxgbfHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/unist": "^3.0.0", + "devlop": "^1.0.0", + "estree-util-visit": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "vfile-message": "^4.0.0" + } + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", + "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/mrmime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz", + "integrity": "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/muggle-string": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/muggle-string/-/muggle-string-0.4.1.tgz", + "integrity": "sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/neotraverse": { + "version": "0.6.18", + "resolved": "https://registry.npmjs.org/neotraverse/-/neotraverse-0.6.18.tgz", + "integrity": "sha512-Z4SmBUweYa09+o6pG+eASabEpP6QkQ70yHj351pQoEXIs8uHbaU2DWVmzBANKgflPa47A50PtB2+NgRpQvr7vA==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/nlcst-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/nlcst-to-string/-/nlcst-to-string-4.0.0.tgz", + "integrity": "sha512-YKLBCcUYKAg0FNlOBT6aI91qFmSiFKiluk655WzPF+DDMA02qIyy8uiRqI8QXtcFpEvll12LpL5MXqEmAZ+dcA==", + "license": "MIT", + "dependencies": { + "@types/nlcst": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/node-fetch-native": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/node-fetch-native/-/node-fetch-native-1.6.7.tgz", + "integrity": "sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==", + "license": "MIT" + }, + "node_modules/node-mock-http": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/node-mock-http/-/node-mock-http-1.0.4.tgz", + "integrity": "sha512-8DY+kFsDkNXy1sJglUfuODx1/opAGJGyrTuFqEoN90oRc2Vk0ZbD4K2qmKXBBEhZQzdKHIVfEJpDU8Ak2NJEvQ==", + "license": "MIT" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/obug": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", + "integrity": "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==", + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT" + }, + "node_modules/ofetch": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/ofetch/-/ofetch-1.5.1.tgz", + "integrity": "sha512-2W4oUZlVaqAPAil6FUg/difl6YhqhUR7x2eZY4bQCko22UXg3hptq9KLQdqFClV+Wu85UX7hNtdGTngi/1BxcA==", + "license": "MIT", + "dependencies": { + "destr": "^2.0.5", + "node-fetch-native": "^1.6.7", + "ufo": "^1.6.1" + } + }, + "node_modules/ohash": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/ohash/-/ohash-2.0.11.tgz", + "integrity": "sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==", + "license": "MIT" + }, + "node_modules/oniguruma-parser": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/oniguruma-parser/-/oniguruma-parser-0.12.2.tgz", + "integrity": "sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==", + "license": "MIT" + }, + "node_modules/oniguruma-to-es": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/oniguruma-to-es/-/oniguruma-to-es-4.3.6.tgz", + "integrity": "sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==", + "license": "MIT", + "dependencies": { + "oniguruma-parser": "^0.12.2", + "regex": "^6.1.0", + "regex-recursion": "^6.0.2" + } + }, + "node_modules/p-limit": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-7.3.0.tgz", + "integrity": "sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^1.2.1" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-queue": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.1.2.tgz", + "integrity": "sha512-ktsDOALzTYTWWF1PbkNVg2rOt+HaOaMWJMUnt7T3qf5tvZ1L8dBW3tObzprBcXNMKkwj+yFSLqHso0x+UFcJXw==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.1", + "p-timeout": "^7.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.1.tgz", + "integrity": "sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-manager-detector": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.6.0.tgz", + "integrity": "sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA==", + "license": "MIT" + }, + "node_modules/pagefind": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/pagefind/-/pagefind-1.5.2.tgz", + "integrity": "sha512-XTUaK0hXMCu2jszWE584JGQT7y284TmMV9l/HX3rnG5uo3rHI/uHU56XTyyyPFjeWEBxECbAi0CaFDJOONtG0Q==", + "license": "MIT", + "bin": { + "pagefind": "lib/runner/bin.cjs" + }, + "optionalDependencies": { + "@pagefind/darwin-arm64": "1.5.2", + "@pagefind/darwin-x64": "1.5.2", + "@pagefind/freebsd-x64": "1.5.2", + "@pagefind/linux-arm64": "1.5.2", + "@pagefind/linux-x64": "1.5.2", + "@pagefind/windows-arm64": "1.5.2", + "@pagefind/windows-x64": "1.5.2" + } + }, + "node_modules/parse-entities": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", + "integrity": "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse-entities/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, + "node_modules/parse-latin": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/parse-latin/-/parse-latin-7.0.0.tgz", + "integrity": "sha512-mhHgobPPua5kZ98EF4HWiH167JWBfl4pvAIXXdbaVohtK7a6YBOy56kvhCqduqyo/f3yrHFWmqmiMg/BkBkYYQ==", + "license": "MIT", + "dependencies": { + "@types/nlcst": "^2.0.0", + "@types/unist": "^3.0.0", + "nlcst-to-string": "^4.0.0", + "unist-util-modify-children": "^4.0.0", + "unist-util-visit-children": "^3.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==", + "dev": true, + "license": "MIT" + }, + "node_modules/piccolore": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/piccolore/-/piccolore-0.1.3.tgz", + "integrity": "sha512-o8bTeDWjE086iwKrROaDf31K0qC/BENdm15/uH9usSC/uZjJOKb2YGiVHfLY4GhwsERiPI1jmwI2XrA7ACOxVw==", + "license": "ISC" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.10", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz", + "integrity": "sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-nested": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/postcss-nested/-/postcss-nested-6.2.0.tgz", + "integrity": "sha512-HQbt28KulC5AJzG+cZtj9kvKB93CFCdLvog1WFLf1D+xmMvPGlBstkpTEZfK5+AN9hfJocyBFCNiqyS48bpgzQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "postcss-selector-parser": "^6.1.1" + }, + "engines": { + "node": ">=12.0" + }, + "peerDependencies": { + "postcss": "^8.2.14" + } + }, + "node_modules/postcss-selector-parser": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", + "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/prettier": { + "version": "3.8.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.3.tgz", + "integrity": "sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/prismjs": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.30.0.tgz", + "integrity": "sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/property-information": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.1.0.tgz", + "integrity": "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/radix3": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/radix3/-/radix3-1.1.2.tgz", + "integrity": "sha512-b484I/7b8rDEdSDKckSSBA8knMpcdsXudlE/LNL639wFoHKwLbEkQFZHWEYwDC0wa0FKUcCY+GAF73Z7wxNVFA==", + "license": "MIT" + }, + "node_modules/readdirp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz", + "integrity": "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==", + "license": "MIT", + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/recma-build-jsx": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/recma-build-jsx/-/recma-build-jsx-1.0.0.tgz", + "integrity": "sha512-8GtdyqaBcDfva+GUKDr3nev3VpKAhup1+RvkMvUxURHpW7QyIvk9F5wz7Vzo06CEMSilw6uArgRqhpiUcWp8ew==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "estree-util-build-jsx": "^3.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/recma-jsx": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/recma-jsx/-/recma-jsx-1.0.1.tgz", + "integrity": "sha512-huSIy7VU2Z5OLv6oFLosQGGDqPqdO1iq6bWNAdhzMxSJP7RAso4fCZ1cKu8j9YHCZf3TPrq4dw3okhrylgcd7w==", + "license": "MIT", + "dependencies": { + "acorn-jsx": "^5.0.0", + "estree-util-to-js": "^2.0.0", + "recma-parse": "^1.0.0", + "recma-stringify": "^1.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + }, + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/recma-parse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/recma-parse/-/recma-parse-1.0.0.tgz", + "integrity": "sha512-OYLsIGBB5Y5wjnSnQW6t3Xg7q3fQ7FWbw/vcXtORTnyaSFscOtABg+7Pnz6YZ6c27fG1/aN8CjfwoUEUIdwqWQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "esast-util-from-js": "^2.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/recma-stringify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/recma-stringify/-/recma-stringify-1.0.0.tgz", + "integrity": "sha512-cjwII1MdIIVloKvC9ErQ+OgAtwHBmcZ0Bg4ciz78FtbT8In39aAYbaA7zvxQ61xVMSPE8WxhLwLbhif4Js2C+g==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "estree-util-to-js": "^2.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/regex/-/regex-6.1.0.tgz", + "integrity": "sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==", + "license": "MIT", + "dependencies": { + "regex-utilities": "^2.3.0" + } + }, + "node_modules/regex-recursion": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/regex-recursion/-/regex-recursion-6.0.2.tgz", + "integrity": "sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==", + "license": "MIT", + "dependencies": { + "regex-utilities": "^2.3.0" + } + }, + "node_modules/regex-utilities": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/regex-utilities/-/regex-utilities-2.3.0.tgz", + "integrity": "sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==", + "license": "MIT" + }, + "node_modules/rehype": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/rehype/-/rehype-13.0.2.tgz", + "integrity": "sha512-j31mdaRFrwFRUIlxGeuPXXKWQxet52RBQRvCmzl5eCefn/KGbomK5GMHNMsOJf55fgo3qw5tST5neDuarDYR2A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "rehype-parse": "^9.0.0", + "rehype-stringify": "^10.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-expressive-code": { + "version": "0.41.7", + "resolved": "https://registry.npmjs.org/rehype-expressive-code/-/rehype-expressive-code-0.41.7.tgz", + "integrity": "sha512-25f8ZMSF1d9CMscX7Cft0TSQIqdwjce2gDOvQ+d/w0FovsMwrSt3ODP4P3Z7wO1jsIJ4eYyaDRnIR/27bd/EMQ==", + "license": "MIT", + "dependencies": { + "expressive-code": "^0.41.7" + } + }, + "node_modules/rehype-format": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/rehype-format/-/rehype-format-5.0.1.tgz", + "integrity": "sha512-zvmVru9uB0josBVpr946OR8ui7nJEdzZobwLOOqHb/OOD88W0Vk2SqLwoVOj0fM6IPCCO6TaV9CvQvJMWwukFQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-format": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-parse": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/rehype-parse/-/rehype-parse-9.0.1.tgz", + "integrity": "sha512-ksCzCD0Fgfh7trPDxr2rSylbwq9iYDkSn8TCDmEJ49ljEUBxDVCzCHv7QNzZOfODanX4+bWQ4WZqLCRWYLfhag==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-from-html": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-raw": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/rehype-raw/-/rehype-raw-7.0.0.tgz", + "integrity": "sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-raw": "^9.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-recma": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/rehype-recma/-/rehype-recma-1.0.0.tgz", + "integrity": "sha512-lqA4rGUf1JmacCNWWZx0Wv1dHqMwxzsDWYMTowuplHF3xH0N/MmrZ/G3BDZnzAkRmxDadujCjaKM2hqYdCBOGw==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/hast": "^3.0.0", + "hast-util-to-estree": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-stringify": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/rehype-stringify/-/rehype-stringify-10.0.1.tgz", + "integrity": "sha512-k9ecfXHmIPuFVI61B9DeLPN0qFHfawM6RsuX48hoqlaKSF61RskNjSm1lI8PhBEM0MRdLxVVm4WmTqJQccH9mA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-to-html": "^9.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-directive": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/remark-directive/-/remark-directive-3.0.1.tgz", + "integrity": "sha512-gwglrEQEZcZYgVyG1tQuA+h58EZfq5CSULw7J90AFuCTyib1thgHPoqQ+h9iFvU6R+vnZ5oNFQR5QKgGpk741A==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-directive": "^3.0.0", + "micromark-extension-directive": "^3.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-gfm": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.1.tgz", + "integrity": "sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-gfm": "^3.0.0", + "micromark-extension-gfm": "^3.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-mdx": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/remark-mdx/-/remark-mdx-3.1.1.tgz", + "integrity": "sha512-Pjj2IYlUY3+D8x00UJsIOg5BEvfMyeI+2uLPn9VO9Wg4MEtN/VTIq2NEJQfde9PnX15KgtHyl9S0BcTnWrIuWg==", + "license": "MIT", + "dependencies": { + "mdast-util-mdx": "^3.0.0", + "micromark-extension-mdxjs": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-parse": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", + "integrity": "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-rehype": { + "version": "11.1.2", + "resolved": "https://registry.npmjs.org/remark-rehype/-/remark-rehype-11.1.2.tgz", + "integrity": "sha512-Dh7l57ianaEoIpzbp0PC9UKAdCSVklD8E5Rpw7ETfbTl3FqcOOgq5q2LVDhgGCkaBv7p24JXikPdvhhmHvKMsw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "mdast-util-to-hast": "^13.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-smartypants": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/remark-smartypants/-/remark-smartypants-3.0.2.tgz", + "integrity": "sha512-ILTWeOriIluwEvPjv67v7Blgrcx+LZOkAUVtKI3putuhlZm84FnqDORNXPPm+HY3NdZOMhyDwZ1E+eZB/Df5dA==", + "license": "MIT", + "dependencies": { + "retext": "^9.0.0", + "retext-smartypants": "^6.0.0", + "unified": "^11.0.4", + "unist-util-visit": "^5.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/remark-stringify": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-11.0.0.tgz", + "integrity": "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-to-markdown": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/request-light": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/request-light/-/request-light-0.7.0.tgz", + "integrity": "sha512-lMbBMrDoxgsyO+yB3sDcrDuX85yYt7sS8BfQd11jtbW/z5ZWgLZRcEGLsLoYw7I0WSUGQBs8CC8ScIxkTX1+6Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/retext": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/retext/-/retext-9.0.0.tgz", + "integrity": "sha512-sbMDcpHCNjvlheSgMfEcVrZko3cDzdbe1x/e7G66dFp0Ff7Mldvi2uv6JkJQzdRcvLYE8CA8Oe8siQx8ZOgTcA==", + "license": "MIT", + "dependencies": { + "@types/nlcst": "^2.0.0", + "retext-latin": "^4.0.0", + "retext-stringify": "^4.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/retext-latin": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/retext-latin/-/retext-latin-4.0.0.tgz", + "integrity": "sha512-hv9woG7Fy0M9IlRQloq/N6atV82NxLGveq+3H2WOi79dtIYWN8OaxogDm77f8YnVXJL2VD3bbqowu5E3EMhBYA==", + "license": "MIT", + "dependencies": { + "@types/nlcst": "^2.0.0", + "parse-latin": "^7.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/retext-smartypants": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/retext-smartypants/-/retext-smartypants-6.2.0.tgz", + "integrity": "sha512-kk0jOU7+zGv//kfjXEBjdIryL1Acl4i9XNkHxtM7Tm5lFiCog576fjNC9hjoR7LTKQ0DsPWy09JummSsH1uqfQ==", + "license": "MIT", + "dependencies": { + "@types/nlcst": "^2.0.0", + "nlcst-to-string": "^4.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/retext-stringify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/retext-stringify/-/retext-stringify-4.0.0.tgz", + "integrity": "sha512-rtfN/0o8kL1e+78+uxPTqu1Klt0yPzKuQ2BfWwwfgIUSayyzxpM1PJzkKt4V8803uB9qSy32MvI7Xep9khTpiA==", + "license": "MIT", + "dependencies": { + "@types/nlcst": "^2.0.0", + "nlcst-to-string": "^4.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rollup": { + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.2.tgz", + "integrity": "sha512-J9qZyW++QK/09NyN/zeO0dG/1GdGfyp9lV8ajHnRVLfo/uFsbji5mHnDgn/qYdUHyCkM2N+8VyspgZclfAh0eQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.60.2", + "@rollup/rollup-android-arm64": "4.60.2", + "@rollup/rollup-darwin-arm64": "4.60.2", + "@rollup/rollup-darwin-x64": "4.60.2", + "@rollup/rollup-freebsd-arm64": "4.60.2", + "@rollup/rollup-freebsd-x64": "4.60.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.60.2", + "@rollup/rollup-linux-arm-musleabihf": "4.60.2", + "@rollup/rollup-linux-arm64-gnu": "4.60.2", + "@rollup/rollup-linux-arm64-musl": "4.60.2", + "@rollup/rollup-linux-loong64-gnu": "4.60.2", + "@rollup/rollup-linux-loong64-musl": "4.60.2", + "@rollup/rollup-linux-ppc64-gnu": "4.60.2", + "@rollup/rollup-linux-ppc64-musl": "4.60.2", + "@rollup/rollup-linux-riscv64-gnu": "4.60.2", + "@rollup/rollup-linux-riscv64-musl": "4.60.2", + "@rollup/rollup-linux-s390x-gnu": "4.60.2", + "@rollup/rollup-linux-x64-gnu": "4.60.2", + "@rollup/rollup-linux-x64-musl": "4.60.2", + "@rollup/rollup-openbsd-x64": "4.60.2", + "@rollup/rollup-openharmony-arm64": "4.60.2", + "@rollup/rollup-win32-arm64-msvc": "4.60.2", + "@rollup/rollup-win32-ia32-msvc": "4.60.2", + "@rollup/rollup-win32-x64-gnu": "4.60.2", + "@rollup/rollup-win32-x64-msvc": "4.60.2", + "fsevents": "~2.3.2" + } + }, + "node_modules/sax": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", + "integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=11.0.0" + } + }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, + "node_modules/shiki": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-4.0.2.tgz", + "integrity": "sha512-eAVKTMedR5ckPo4xne/PjYQYrU3qx78gtJZ+sHlXEg5IHhhoQhMfZVzetTYuaJS0L2Ef3AcCRzCHV8T0WI6nIQ==", + "license": "MIT", + "dependencies": { + "@shikijs/core": "4.0.2", + "@shikijs/engine-javascript": "4.0.2", + "@shikijs/engine-oniguruma": "4.0.2", + "@shikijs/langs": "4.0.2", + "@shikijs/themes": "4.0.2", + "@shikijs/types": "4.0.2", + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.4" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "license": "MIT" + }, + "node_modules/sitemap": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/sitemap/-/sitemap-9.0.1.tgz", + "integrity": "sha512-S6hzjGJSG3d6if0YoF5kTyeRJvia6FSTBroE5fQ0bu1QNxyJqhhinfUsXi9fH3MgtXODWvwo2BDyQSnhPQ88uQ==", + "license": "MIT", + "dependencies": { + "@types/node": "^24.9.2", + "@types/sax": "^1.2.1", + "arg": "^5.0.0", + "sax": "^1.4.1" + }, + "bin": { + "sitemap": "dist/esm/cli.js" + }, + "engines": { + "node": ">=20.19.5", + "npm": ">=10.8.2" + } + }, + "node_modules/smol-toml": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.1.tgz", + "integrity": "sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==", + "license": "BSD-3-Clause", + "engines": { + "node": ">= 18" + }, + "funding": { + "url": "https://github.com/sponsors/cyyynthia" + } + }, + "node_modules/source-map": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.6.tgz", + "integrity": "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==", + "license": "BSD-3-Clause", + "engines": { + "node": ">= 12" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/space-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", + "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/starlight-theme-rapide": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/starlight-theme-rapide/-/starlight-theme-rapide-0.5.2.tgz", + "integrity": "sha512-4kNJutt0/3E0rjZvG9hvKeTTHGOHC97t2ZQDwDcokTM25yLgxavFu7BCm2Dm2UTw7fJpRXpu4gUWJ9xAOBmNww==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@astrojs/starlight": ">=0.34.0" + } + }, + "node_modules/stream-replace-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/stream-replace-string/-/stream-replace-string-2.0.0.tgz", + "integrity": "sha512-TlnjJ1C0QrmxRNrON00JvaFFlNh5TTG00APw23j74ET7gkQpTASi6/L2fuiav8pzK715HXtUeClpBTw2NPSn6w==", + "license": "MIT" + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/stringify-entities": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", + "integrity": "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==", + "license": "MIT", + "dependencies": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^3.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/style-to-js": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/style-to-js/-/style-to-js-1.1.21.tgz", + "integrity": "sha512-RjQetxJrrUJLQPHbLku6U/ocGtzyjbJMP9lCNK7Ag0CNh690nSH8woqWH9u16nMjYBAok+i7JO1NP2pOy8IsPQ==", + "license": "MIT", + "dependencies": { + "style-to-object": "1.0.14" + } + }, + "node_modules/style-to-object": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-1.0.14.tgz", + "integrity": "sha512-LIN7rULI0jBscWQYaSswptyderlarFkjQ+t79nzty8tcIAceVomEVlLzH5VP4Cmsv6MtKhs7qaAiwlcp+Mgaxw==", + "license": "MIT", + "dependencies": { + "inline-style-parser": "0.2.7" + } + }, + "node_modules/svgo": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-4.0.1.tgz", + "integrity": "sha512-XDpWUOPC6FEibaLzjfe0ucaV0YrOjYotGJO1WpF0Zd+n6ZGEQUsSugaoLq9QkEZtAfQIxT42UChcssDVPP3+/w==", + "license": "MIT", + "dependencies": { + "commander": "^11.1.0", + "css-select": "^5.1.0", + "css-tree": "^3.0.1", + "css-what": "^6.1.0", + "csso": "^5.0.5", + "picocolors": "^1.1.1", + "sax": "^1.5.0" + }, + "bin": { + "svgo": "bin/svgo.js" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/svgo" + } + }, + "node_modules/tiny-inflate": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tiny-inflate/-/tiny-inflate-1.0.3.tgz", + "integrity": "sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw==", + "license": "MIT" + }, + "node_modules/tinyclip": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/tinyclip/-/tinyclip-0.1.12.tgz", + "integrity": "sha512-Ae3OVUqifDw0wBriIBS7yVaW44Dp6eSHQcyq4Igc7eN2TJH/2YsicswaW+J/OuMvhpDPOKEgpAZCjkb4hpoyeA==", + "license": "MIT", + "engines": { + "node": "^16.14.0 || >= 17.3.0" + } + }, + "node_modules/tinyexec": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.1.1.tgz", + "integrity": "sha512-VKS/ZaQhhkKFMANmAOhhXVoIfBXblQxGX1myCQ2faQrfmobMftXeJPcZGp0gS07ocvGJWDLZGyOZDadDBqYIJg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.16", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", + "integrity": "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==", + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/trim-lines": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", + "integrity": "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/trough": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", + "integrity": "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/tsconfck": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/tsconfck/-/tsconfck-3.1.6.tgz", + "integrity": "sha512-ks6Vjr/jEw0P1gmOVwutM3B7fWxoWBL2KRDb1JfqGVawBmO5UsvmWOQFGHBPl5yxYz4eERr19E6L7NMv+Fej4w==", + "license": "MIT", + "bin": { + "tsconfck": "bin/tsconfck.js" + }, + "engines": { + "node": "^18 || >=20" + }, + "peerDependencies": { + "typescript": "^5.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + }, + "node_modules/typesafe-path": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/typesafe-path/-/typesafe-path-0.2.2.tgz", + "integrity": "sha512-OJabfkAg1WLZSqJAJ0Z6Sdt3utnbzr/jh+NAHoyWHJe8CMSy79Gm085094M9nvTPy22KzTVn5Zq5mbapCI/hPA==", + "dev": true, + "license": "MIT" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "devOptional": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/typescript-auto-import-cache": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/typescript-auto-import-cache/-/typescript-auto-import-cache-0.3.6.tgz", + "integrity": "sha512-RpuHXrknHdVdK7wv/8ug3Fr0WNsNi5l5aB8MYYuXhq2UH5lnEB1htJ1smhtD5VeCsGr2p8mUDtd83LCQDFVgjQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.3.8" + } + }, + "node_modules/ufo": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.3.tgz", + "integrity": "sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==", + "license": "MIT" + }, + "node_modules/ultrahtml": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/ultrahtml/-/ultrahtml-1.6.0.tgz", + "integrity": "sha512-R9fBn90VTJrqqLDwyMph+HGne8eqY1iPfYhPzZrvKpIfwkWZbcYlfpsb8B9dTvBfpy1/hqAD7Wi8EKfP9e8zdw==", + "license": "MIT" + }, + "node_modules/uncrypto": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/uncrypto/-/uncrypto-0.1.3.tgz", + "integrity": "sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==", + "license": "MIT" + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "license": "MIT" + }, + "node_modules/unified": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "bail": "^2.0.0", + "devlop": "^1.0.0", + "extend": "^3.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unifont": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/unifont/-/unifont-0.7.4.tgz", + "integrity": "sha512-oHeis4/xl42HUIeHuNZRGEvxj5AaIKR+bHPNegRq5LV1gdc3jundpONbjglKpihmJf+dswygdMJn3eftGIMemg==", + "license": "MIT", + "dependencies": { + "css-tree": "^3.1.0", + "ofetch": "^1.5.1", + "ohash": "^2.0.11" + } + }, + "node_modules/unist-util-find-after": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-find-after/-/unist-util-find-after-5.0.0.tgz", + "integrity": "sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-is": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.1.tgz", + "integrity": "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-modify-children": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-modify-children/-/unist-util-modify-children-4.0.0.tgz", + "integrity": "sha512-+tdN5fGNddvsQdIzUF3Xx82CU9sMM+fA0dLgR9vOmT0oPT2jH+P1nd5lSqfCfXAw+93NhcXNY2qqvTUtE4cQkw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "array-iterate": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-5.0.0.tgz", + "integrity": "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position-from-estree": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position-from-estree/-/unist-util-position-from-estree-2.0.0.tgz", + "integrity": "sha512-KaFVRjoqLyF6YXCbVLNad/eS4+OfPQQn2yOd7zF/h5T/CSL2v8NpN6a5TPvtbXthAGw5nG+PuTtq+DdIZr+cRQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-remove-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-5.0.0.tgz", + "integrity": "sha512-Hp5Kh3wLxv0PHj9m2yZhhLt58KzPtEYKQQ4yxfYFEO7EvHwzyDYnduhHnY1mDxoqr7VUwVuHXk9RXKIiYS1N8Q==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.1.0.tgz", + "integrity": "sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-children": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unist-util-visit-children/-/unist-util-visit-children-3.0.0.tgz", + "integrity": "sha512-RgmdTfSBOg04sdPcpTSD1jzoNBjt9a80/ZCzp5cI9n1qPzLZWF9YdvWGN2zmTumP1HWhXKdUWexjy/Wy/lJ7tA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.2.tgz", + "integrity": "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unstorage": { + "version": "1.17.5", + "resolved": "https://registry.npmjs.org/unstorage/-/unstorage-1.17.5.tgz", + "integrity": "sha512-0i3iqvRfx29hkNntHyQvJTpf5W9dQ9ZadSoRU8+xVlhVtT7jAX57fazYO9EHvcRCfBCyi5YRya7XCDOsbTgkPg==", + "license": "MIT", + "dependencies": { + "anymatch": "^3.1.3", + "chokidar": "^5.0.0", + "destr": "^2.0.5", + "h3": "^1.15.10", + "lru-cache": "^11.2.7", + "node-fetch-native": "^1.6.7", + "ofetch": "^1.5.1", + "ufo": "^1.6.3" + }, + "peerDependencies": { + "@azure/app-configuration": "^1.8.0", + "@azure/cosmos": "^4.2.0", + "@azure/data-tables": "^13.3.0", + "@azure/identity": "^4.6.0", + "@azure/keyvault-secrets": "^4.9.0", + "@azure/storage-blob": "^12.26.0", + "@capacitor/preferences": "^6 || ^7 || ^8", + "@deno/kv": ">=0.9.0", + "@netlify/blobs": "^6.5.0 || ^7.0.0 || ^8.1.0 || ^9.0.0 || ^10.0.0", + "@planetscale/database": "^1.19.0", + "@upstash/redis": "^1.34.3", + "@vercel/blob": ">=0.27.1", + "@vercel/functions": "^2.2.12 || ^3.0.0", + "@vercel/kv": "^1 || ^2 || ^3", + "aws4fetch": "^1.0.20", + "db0": ">=0.2.1", + "idb-keyval": "^6.2.1", + "ioredis": "^5.4.2", + "uploadthing": "^7.4.4" + }, + "peerDependenciesMeta": { + "@azure/app-configuration": { + "optional": true + }, + "@azure/cosmos": { + "optional": true + }, + "@azure/data-tables": { + "optional": true + }, + "@azure/identity": { + "optional": true + }, + "@azure/keyvault-secrets": { + "optional": true + }, + "@azure/storage-blob": { + "optional": true + }, + "@capacitor/preferences": { + "optional": true + }, + "@deno/kv": { + "optional": true + }, + "@netlify/blobs": { + "optional": true + }, + "@planetscale/database": { + "optional": true + }, + "@upstash/redis": { + "optional": true + }, + "@vercel/blob": { + "optional": true + }, + "@vercel/functions": { + "optional": true + }, + "@vercel/kv": { + "optional": true + }, + "aws4fetch": { + "optional": true + }, + "db0": { + "optional": true + }, + "idb-keyval": { + "optional": true + }, + "ioredis": { + "optional": true + }, + "uploadthing": { + "optional": true + } + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/vfile": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", + "integrity": "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-location": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-5.0.3.tgz", + "integrity": "sha512-5yXvWDEgqeiYiBe1lbxYF7UMAIm/IcopxMHrMQDq3nvKcjPKIhZklUKL+AE7J7uApI4kwe2snsK+eI6UTj9EHg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.3.tgz", + "integrity": "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vite": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.2.tgz", + "integrity": "sha512-Bby3NOsna2jsjfLVOHKes8sGwgl4TT0E6vvpYgnAYDIF/tie7MRaFthmKuHx1NSXjiTueXH3do80FMQgvEktRg==", + "license": "MIT", + "dependencies": { + "esbuild": "^0.27.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitefu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-1.1.3.tgz", + "integrity": "sha512-ub4okH7Z5KLjb6hDyjqrGXqWtWvoYdU3IGm/NorpgHncKoLTCfRIbvlhBm7r0YstIaQRYlp4yEbFqDcKSzXSSg==", + "license": "MIT", + "workspaces": [ + "tests/deps/*", + "tests/projects/*", + "tests/projects/workspace/packages/*" + ], + "peerDependencies": { + "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "vite": { + "optional": true + } + } + }, + "node_modules/volar-service-css": { + "version": "0.0.70", + "resolved": "https://registry.npmjs.org/volar-service-css/-/volar-service-css-0.0.70.tgz", + "integrity": "sha512-K1qyOvBpE3rzdAv3e4/6Rv5yizrYPy5R/ne3IWCAzLBuMO4qBMV3kSqWzj6KUVe6S0AnN6wxF7cRkiaKfYMYJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "vscode-css-languageservice": "^6.3.0", + "vscode-languageserver-textdocument": "^1.0.11", + "vscode-uri": "^3.0.8" + }, + "peerDependencies": { + "@volar/language-service": "~2.4.0" + }, + "peerDependenciesMeta": { + "@volar/language-service": { + "optional": true + } + } + }, + "node_modules/volar-service-emmet": { + "version": "0.0.70", + "resolved": "https://registry.npmjs.org/volar-service-emmet/-/volar-service-emmet-0.0.70.tgz", + "integrity": "sha512-xi5bC4m/VyE3zy/n2CXspKeDZs3qA41tHLTw275/7dNWM/RqE2z3BnDICQybHIVp/6G1iOQj5c1qXMgQC08TNg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@emmetio/css-parser": "^0.4.1", + "@emmetio/html-matcher": "^1.3.0", + "@vscode/emmet-helper": "^2.9.3", + "vscode-uri": "^3.0.8" + }, + "peerDependencies": { + "@volar/language-service": "~2.4.0" + }, + "peerDependenciesMeta": { + "@volar/language-service": { + "optional": true + } + } + }, + "node_modules/volar-service-html": { + "version": "0.0.70", + "resolved": "https://registry.npmjs.org/volar-service-html/-/volar-service-html-0.0.70.tgz", + "integrity": "sha512-eR6vCgMdmYAo4n+gcT7DSyBQbwB8S3HZZvSagTf0sxNaD4WppMCFfpqWnkrlGStPKMZvMiejRRVmqsX9dYcTvQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "vscode-html-languageservice": "^5.3.0", + "vscode-languageserver-textdocument": "^1.0.11", + "vscode-uri": "^3.0.8" + }, + "peerDependencies": { + "@volar/language-service": "~2.4.0" + }, + "peerDependenciesMeta": { + "@volar/language-service": { + "optional": true + } + } + }, + "node_modules/volar-service-prettier": { + "version": "0.0.70", + "resolved": "https://registry.npmjs.org/volar-service-prettier/-/volar-service-prettier-0.0.70.tgz", + "integrity": "sha512-Z6BCFSpGVCd8BPAsZ785Kce1BGlWd5ODqmqZGVuB14MJvrR4+CYz6cDy4F+igmE1gMifqfvMhdgT8Aud4M5ngg==", + "dev": true, + "license": "MIT", + "dependencies": { + "vscode-uri": "^3.0.8" + }, + "peerDependencies": { + "@volar/language-service": "~2.4.0", + "prettier": "^2.2 || ^3.0" + }, + "peerDependenciesMeta": { + "@volar/language-service": { + "optional": true + }, + "prettier": { + "optional": true + } + } + }, + "node_modules/volar-service-typescript": { + "version": "0.0.70", + "resolved": "https://registry.npmjs.org/volar-service-typescript/-/volar-service-typescript-0.0.70.tgz", + "integrity": "sha512-l46Bx4cokkUedTd74ojO5H/zqHZJ8SUuyZ0IB8JN4jfRqUM3bQFBHoOwlZCyZmOeO0A3RQNkMnFclxO4c++gsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-browserify": "^1.0.1", + "semver": "^7.6.2", + "typescript-auto-import-cache": "^0.3.5", + "vscode-languageserver-textdocument": "^1.0.11", + "vscode-nls": "^5.2.0", + "vscode-uri": "^3.0.8" + }, + "peerDependencies": { + "@volar/language-service": "~2.4.0" + }, + "peerDependenciesMeta": { + "@volar/language-service": { + "optional": true + } + } + }, + "node_modules/volar-service-typescript-twoslash-queries": { + "version": "0.0.70", + "resolved": "https://registry.npmjs.org/volar-service-typescript-twoslash-queries/-/volar-service-typescript-twoslash-queries-0.0.70.tgz", + "integrity": "sha512-IdD13Z9N2Bu8EM6CM0fDV1E69olEYGHDU25X51YXmq8Y0CmJ2LNj6gOiBJgpS5JGUqFzECVhMNBW7R0sPdRTMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "vscode-uri": "^3.0.8" + }, + "peerDependencies": { + "@volar/language-service": "~2.4.0" + }, + "peerDependenciesMeta": { + "@volar/language-service": { + "optional": true + } + } + }, + "node_modules/volar-service-yaml": { + "version": "0.0.70", + "resolved": "https://registry.npmjs.org/volar-service-yaml/-/volar-service-yaml-0.0.70.tgz", + "integrity": "sha512-0c8bXDBeoATF9F6iPIlOuYTuZAC4c+yi0siQo920u7eiBJk8oQmUmg9cDUbR4+Gl++bvGP4plj3fErbJuPqdcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "vscode-uri": "^3.0.8", + "yaml-language-server": "~1.20.0" + }, + "peerDependencies": { + "@volar/language-service": "~2.4.0" + }, + "peerDependenciesMeta": { + "@volar/language-service": { + "optional": true + } + } + }, + "node_modules/vscode-css-languageservice": { + "version": "6.3.10", + "resolved": "https://registry.npmjs.org/vscode-css-languageservice/-/vscode-css-languageservice-6.3.10.tgz", + "integrity": "sha512-eq5N9Er3fC4vA9zd9EFhyBG90wtCCuXgRSpAndaOgXMh1Wgep5lBgRIeDgjZBW9pa+332yC9+49cZMW8jcL3MA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vscode/l10n": "^0.0.18", + "vscode-languageserver-textdocument": "^1.0.12", + "vscode-languageserver-types": "3.17.5", + "vscode-uri": "^3.1.0" + } + }, + "node_modules/vscode-html-languageservice": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/vscode-html-languageservice/-/vscode-html-languageservice-5.6.2.tgz", + "integrity": "sha512-ulCrSnFnfQ16YzvwnYUgEbUEl/ZG7u2eV27YhvLObSHKkb8fw1Z9cgsnUwjTEeDIdJDoTDTDpxuhQwoenoLNMg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vscode/l10n": "^0.0.18", + "vscode-languageserver-textdocument": "^1.0.12", + "vscode-languageserver-types": "^3.17.5", + "vscode-uri": "^3.1.0" + } + }, + "node_modules/vscode-json-languageservice": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/vscode-json-languageservice/-/vscode-json-languageservice-4.1.8.tgz", + "integrity": "sha512-0vSpg6Xd9hfV+eZAaYN63xVVMOTmJ4GgHxXnkLCh+9RsQBkWKIghzLhW2B9ebfG+LQQg8uLtsQ2aUKjTgE+QOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "jsonc-parser": "^3.0.0", + "vscode-languageserver-textdocument": "^1.0.1", + "vscode-languageserver-types": "^3.16.0", + "vscode-nls": "^5.0.0", + "vscode-uri": "^3.0.2" + }, + "engines": { + "npm": ">=7.0.0" + } + }, + "node_modules/vscode-json-languageservice/node_modules/jsonc-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", + "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/vscode-jsonrpc": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz", + "integrity": "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/vscode-languageserver": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/vscode-languageserver/-/vscode-languageserver-9.0.1.tgz", + "integrity": "sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "vscode-languageserver-protocol": "3.17.5" + }, + "bin": { + "installServerIntoExtension": "bin/installServerIntoExtension" + } + }, + "node_modules/vscode-languageserver-protocol": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.17.5.tgz", + "integrity": "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==", + "dev": true, + "license": "MIT", + "dependencies": { + "vscode-jsonrpc": "8.2.0", + "vscode-languageserver-types": "3.17.5" + } + }, + "node_modules/vscode-languageserver-textdocument": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.12.tgz", + "integrity": "sha512-cxWNPesCnQCcMPeenjKKsOCKQZ/L6Tv19DTRIGuLWe32lyzWhihGVJ/rcckZXJxfdKCFvRLS3fpBIsV/ZGX4zA==", + "dev": true, + "license": "MIT" + }, + "node_modules/vscode-languageserver-types": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==", + "dev": true, + "license": "MIT" + }, + "node_modules/vscode-nls": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-5.2.0.tgz", + "integrity": "sha512-RAaHx7B14ZU04EU31pT+rKz2/zSl7xMsfIZuo8pd+KZO6PXtQmpevpq3vxvWNcrGbdmhM/rr5Uw5Mz+NBfhVng==", + "dev": true, + "license": "MIT" + }, + "node_modules/vscode-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.1.0.tgz", + "integrity": "sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/web-namespaces": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.1.tgz", + "integrity": "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/which-pm-runs": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.1.0.tgz", + "integrity": "sha512-n1brCuqClxfFfq/Rb0ICg9giSZqCS+pLtccdag6C2HyufBrh3fBOiy9nb6ggRMvWOVH5GrdJskj5iGTZNxd7SA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/xxhash-wasm": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.1.0.tgz", + "integrity": "sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==", + "license": "MIT" + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "devOptional": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/yaml-language-server": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/yaml-language-server/-/yaml-language-server-1.20.0.tgz", + "integrity": "sha512-qhjK/bzSRZ6HtTvgeFvjNPJGWdZ0+x5NREV/9XZWFjIGezew2b4r5JPy66IfOhd5OA7KeFwk1JfmEbnTvev0cA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vscode/l10n": "^0.0.18", + "ajv": "^8.17.1", + "ajv-draft-04": "^1.0.0", + "prettier": "^3.5.0", + "request-light": "^0.5.7", + "vscode-json-languageservice": "4.1.8", + "vscode-languageserver": "^9.0.0", + "vscode-languageserver-textdocument": "^1.0.1", + "vscode-languageserver-types": "^3.16.0", + "vscode-uri": "^3.0.2", + "yaml": "2.7.1" + }, + "bin": { + "yaml-language-server": "bin/yaml-language-server" + } + }, + "node_modules/yaml-language-server/node_modules/request-light": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/request-light/-/request-light-0.5.8.tgz", + "integrity": "sha512-3Zjgh+8b5fhRJBQZoy+zbVKpAQGLyka0MPgW3zruTF4dFFJ8Fqcfu9YsAvi/rvdcaTeWG3MkbZv4WKxAn/84Lg==", + "dev": true, + "license": "MIT" + }, + "node_modules/yaml-language-server/node_modules/yaml": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.7.1.tgz", + "integrity": "sha512-10ULxpnOCQXxJvBgxsn9ptjq6uviG/htZKk9veJGhlqn3w/DxQ631zFF+nlQXLwmImeS5amR2dl2U8sg6U9jsQ==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "license": "ISC", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/yargs/node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yocto-queue": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", + "license": "MIT", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", + "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + } + } +} diff --git a/docs/package.json b/docs/package.json new file mode 100644 index 0000000000..70bb81a718 --- /dev/null +++ b/docs/package.json @@ -0,0 +1,23 @@ +{ + "name": "awesome-aurora", + "type": "module", + "version": "0.0.1", + "scripts": { + "dev": "astro dev", + "start": "astro dev", + "build": "astro build", + "preview": "astro preview", + "check": "astro check", + "astro": "astro" + }, + "dependencies": { + "@astrojs/starlight": "^0.38.4", + "astro": "^6.1.9", + "sharp": "^0.34.5", + "starlight-theme-rapide": "^0.5.2" + }, + "devDependencies": { + "@astrojs/check": "^0.9.9", + "typescript": "^5.9.3" + } +} diff --git a/docs/public/apple-touch-icon.png b/docs/public/apple-touch-icon.png new file mode 100644 index 0000000000..9162ea8ac9 Binary files /dev/null and b/docs/public/apple-touch-icon.png differ diff --git a/docs/public/favicon-16x16.png b/docs/public/favicon-16x16.png new file mode 100644 index 0000000000..899eabe589 Binary files /dev/null and b/docs/public/favicon-16x16.png differ diff --git a/docs/public/favicon-32x32.png b/docs/public/favicon-32x32.png new file mode 100644 index 0000000000..f6e14fe522 Binary files /dev/null and b/docs/public/favicon-32x32.png differ diff --git a/docs/public/favicon.ico b/docs/public/favicon.ico new file mode 100644 index 0000000000..8386411e9c Binary files /dev/null and b/docs/public/favicon.ico differ diff --git a/docs/public/favicon.svg b/docs/public/favicon.svg new file mode 100644 index 0000000000..e54f04c7b8 --- /dev/null +++ b/docs/public/favicon.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/docs/public/icon-192.png b/docs/public/icon-192.png new file mode 100644 index 0000000000..198cf46707 Binary files /dev/null and b/docs/public/icon-192.png differ diff --git a/docs/public/icon-512.png b/docs/public/icon-512.png new file mode 100644 index 0000000000..89f9a51428 Binary files /dev/null and b/docs/public/icon-512.png differ diff --git a/docs/public/site.webmanifest b/docs/public/site.webmanifest new file mode 100644 index 0000000000..022c2b9166 --- /dev/null +++ b/docs/public/site.webmanifest @@ -0,0 +1,18 @@ +{ + "name": "xLLM", + "short_name": "xLLM", + "icons": [ + { + "src": "/icon-192.png", + "sizes": "192x192", + "type": "image/png" + }, + { + "src": "/icon-512.png", + "sizes": "512x512", + "type": "image/png" + } + ], + "theme_color": "#ffffff", + "background_color": "#ffffff" +} diff --git a/docs/src/assets/logo_with_llm.png b/docs/src/assets/logo_with_llm.png new file mode 100644 index 0000000000..ee61bdf9b4 Binary files /dev/null and b/docs/src/assets/logo_with_llm.png differ diff --git a/docs/src/components/Favicons.astro b/docs/src/components/Favicons.astro new file mode 100644 index 0000000000..5a022f2b7c --- /dev/null +++ b/docs/src/components/Favicons.astro @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/docs/src/components/Head.astro b/docs/src/components/Head.astro new file mode 100644 index 0000000000..ab1410fede --- /dev/null +++ b/docs/src/components/Head.astro @@ -0,0 +1,7 @@ +--- +import DefaultHead from '@astrojs/starlight/components/Head.astro'; +import Favicons from './Favicons.astro'; +--- + + + diff --git a/docs/src/components/Header.astro b/docs/src/components/Header.astro new file mode 100644 index 0000000000..37a6103718 --- /dev/null +++ b/docs/src/components/Header.astro @@ -0,0 +1,374 @@ +--- +import config from 'virtual:starlight/user-config'; + +import Search from 'virtual:starlight/components/Search'; +import SiteTitle from 'virtual:starlight/components/SiteTitle'; +import SocialIcons from 'virtual:starlight/components/SocialIcons'; +import ThemeSelect from 'virtual:starlight/components/ThemeSelect'; +import type { StarlightRouteData } from '@astrojs/starlight/route-data'; + +type SidebarEntry = StarlightRouteData['sidebar'][number]; + +const { sidebar } = Astro.locals.starlightRoute; +const currentLocale = Astro.locals.starlightRoute.locale; + +const shouldRenderSearch = + config.pagefind || config.components.Search !== '@astrojs/starlight/components/Search.astro'; + +const getFirstHref = (entry: SidebarEntry): string | undefined => { + if (entry.type === 'link') return entry.href; + + for (const child of entry.entries) { + const href = getFirstHref(child); + if (href) return href; + } +}; + +const isCurrentEntry = (entry: SidebarEntry): boolean => { + if (entry.type === 'link') return entry.isCurrent; + + return entry.entries.some(isCurrentEntry); +}; + +const isHomeHref = (href: string): boolean => { + const url = new URL(href, Astro.url); + const normalizedPathname = url.pathname.replace(/\/+$/, ''); + const currentLocalePath = currentLocale ? `/${currentLocale}` : ''; + + return normalizedPathname === '' || normalizedPathname === currentLocalePath; +}; + +const navItems = sidebar + .map((entry) => ({ label: entry.label, href: getFirstHref(entry), isCurrent: isCurrentEntry(entry) })) + .filter((entry) => entry.href && !isHomeHref(entry.href)); + +const localeEntries = Object.entries(config.locales ?? {}).map(([code, locale]) => ({ + code, + label: locale?.label ?? code, + href: getLocalizedPathname(code), + isCurrent: code === currentLocale, +})); + +function getLocalizedPathname(locale: string): string { + const url = new URL(Astro.url); + const pathSegments = url.pathname.split('/'); + const currentLocaleIndex = pathSegments.findIndex((segment) => segment in (config.locales ?? {})); + + if (currentLocaleIndex === -1) { + pathSegments.splice(1, 0, locale); + } else { + pathSegments[currentLocaleIndex] = locale; + } + + return pathSegments.join('/') + url.search + url.hash; +} +--- + +
+
+ +
+ +
+ {shouldRenderSearch &&
} +
+ + + { + config.isMultilingual && ( +
+ + + {localeEntries.find((locale) => locale.isCurrent)?.label} + + + +
+ ) + } +
+
+
+ + + + diff --git a/docs/src/components/PageTitle.astro b/docs/src/components/PageTitle.astro new file mode 100644 index 0000000000..7744b8ff46 --- /dev/null +++ b/docs/src/components/PageTitle.astro @@ -0,0 +1,182 @@ +--- +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const { entry } = Astro.locals.starlightRoute; +const title = entry.data.title; +const sourcePath = entry.filePath ? join(process.cwd(), entry.filePath) : undefined; +const sourceMarkdown = + sourcePath && existsSync(sourcePath) ? readFileSync(sourcePath, 'utf-8') : ''; +const bodyMarkdown = stripFrontmatter(sourceMarkdown).trim(); +const copyMarkdown = [`# ${title}`, bodyMarkdown].filter(Boolean).join('\n\n') + '\n'; +const copyPayload = JSON.stringify(copyMarkdown).replace(/ +

{title}

+
+ + +
+ + + + + diff --git a/docs/src/components/SiteTitle.astro b/docs/src/components/SiteTitle.astro new file mode 100644 index 0000000000..b4cedd56e6 --- /dev/null +++ b/docs/src/components/SiteTitle.astro @@ -0,0 +1,42 @@ +--- +import { Image } from 'astro:assets'; +import config from 'virtual:starlight/user-config'; +import logo from '../assets/logo_with_llm.png'; + +const { siteTitle, siteTitleHref } = Astro.locals.starlightRoute; +const alt = config.logo?.alt ?? siteTitle; +--- + + + {alt} + + {siteTitle} + + + + diff --git a/docs/src/content.config.ts b/docs/src/content.config.ts new file mode 100644 index 0000000000..d9ee8c9d1c --- /dev/null +++ b/docs/src/content.config.ts @@ -0,0 +1,7 @@ +import { defineCollection } from 'astro:content'; +import { docsLoader } from '@astrojs/starlight/loaders'; +import { docsSchema } from '@astrojs/starlight/schema'; + +export const collections = { + docs: defineCollection({ loader: docsLoader(), schema: docsSchema() }), +}; diff --git a/docs/src/content/docs/assets/async_schedule_architecture.jpg b/docs/src/content/docs/assets/async_schedule_architecture.jpg new file mode 100644 index 0000000000..6df2571aa5 Binary files /dev/null and b/docs/src/content/docs/assets/async_schedule_architecture.jpg differ diff --git a/docs/src/content/docs/assets/eplb_architecture.png b/docs/src/content/docs/assets/eplb_architecture.png new file mode 100644 index 0000000000..f4db56681f Binary files /dev/null and b/docs/src/content/docs/assets/eplb_architecture.png differ diff --git a/docs/src/content/docs/assets/fixed_steps_scheduler_orca.png b/docs/src/content/docs/assets/fixed_steps_scheduler_orca.png new file mode 100644 index 0000000000..883705d724 Binary files /dev/null and b/docs/src/content/docs/assets/fixed_steps_scheduler_orca.png differ diff --git a/docs/src/content/docs/assets/generative_recommendation_beam_search.png b/docs/src/content/docs/assets/generative_recommendation_beam_search.png new file mode 100644 index 0000000000..70a1b14572 Binary files /dev/null and b/docs/src/content/docs/assets/generative_recommendation_beam_search.png differ diff --git a/docs/src/content/docs/assets/generative_recommendation_integration_architecture.jpg b/docs/src/content/docs/assets/generative_recommendation_integration_architecture.jpg new file mode 100644 index 0000000000..9f9d3c583c Binary files /dev/null and b/docs/src/content/docs/assets/generative_recommendation_integration_architecture.jpg differ diff --git a/docs/src/content/docs/assets/generative_recommendation_integration_architecture_en.svg b/docs/src/content/docs/assets/generative_recommendation_integration_architecture_en.svg new file mode 100644 index 0000000000..ea877091ba --- /dev/null +++ b/docs/src/content/docs/assets/generative_recommendation_integration_architecture_en.svg @@ -0,0 +1,54 @@ + + + + + + + + + Generative Recommendation Integration Architecture + + + Predictor Side + + Sparse feature processing + + Sample construction / feature assembly + + Online service integration + Recall, ranking, and request orchestration + + + Shared Library Integration (.so) + Model body subgraph handoff + Prompt / tokens / tensors + Context handoff + runtime options + Low-overhead path without extra RPC + + + xLLM Side + + LLM main-body inference + + Operators / KV cache / scheduling + + Candidate expansion and response generation + REC runtime path + + + + + subgraph offload + runtime execution + diff --git a/docs/src/content/docs/assets/generative_recommendation_model_onerec.png b/docs/src/content/docs/assets/generative_recommendation_model_onerec.png new file mode 100644 index 0000000000..cc83d97854 Binary files /dev/null and b/docs/src/content/docs/assets/generative_recommendation_model_onerec.png differ diff --git a/docs/src/content/docs/assets/generative_recommendation_model_onetrans.png b/docs/src/content/docs/assets/generative_recommendation_model_onetrans.png new file mode 100644 index 0000000000..07dbb070a4 Binary files /dev/null and b/docs/src/content/docs/assets/generative_recommendation_model_onetrans.png differ diff --git a/docs/src/content/docs/assets/generative_recommendation_overview.png b/docs/src/content/docs/assets/generative_recommendation_overview.png new file mode 100644 index 0000000000..1320e16371 Binary files /dev/null and b/docs/src/content/docs/assets/generative_recommendation_overview.png differ diff --git a/docs/src/content/docs/assets/globalkvcache_architecture.png b/docs/src/content/docs/assets/globalkvcache_architecture.png new file mode 100644 index 0000000000..e9032f421f Binary files /dev/null and b/docs/src/content/docs/assets/globalkvcache_architecture.png differ diff --git a/docs/src/content/docs/assets/graph_mode.png b/docs/src/content/docs/assets/graph_mode.png new file mode 100644 index 0000000000..7a21ddf78d Binary files /dev/null and b/docs/src/content/docs/assets/graph_mode.png differ diff --git a/docs/src/content/docs/assets/logo.png b/docs/src/content/docs/assets/logo.png new file mode 100644 index 0000000000..2c44edf524 Binary files /dev/null and b/docs/src/content/docs/assets/logo.png differ diff --git a/docs/src/content/docs/assets/logo_with_llm.png b/docs/src/content/docs/assets/logo_with_llm.png new file mode 100644 index 0000000000..ee61bdf9b4 Binary files /dev/null and b/docs/src/content/docs/assets/logo_with_llm.png differ diff --git a/docs/src/content/docs/assets/moe_eplevel1.jpg b/docs/src/content/docs/assets/moe_eplevel1.jpg new file mode 100644 index 0000000000..218f79ac20 Binary files /dev/null and b/docs/src/content/docs/assets/moe_eplevel1.jpg differ diff --git a/docs/src/content/docs/assets/moe_eplevel2.jpg b/docs/src/content/docs/assets/moe_eplevel2.jpg new file mode 100644 index 0000000000..643f4d7523 Binary files /dev/null and b/docs/src/content/docs/assets/moe_eplevel2.jpg differ diff --git a/docs/src/content/docs/assets/multi_streams_architecture.jpg b/docs/src/content/docs/assets/multi_streams_architecture.jpg new file mode 100644 index 0000000000..161020e60f Binary files /dev/null and b/docs/src/content/docs/assets/multi_streams_architecture.jpg differ diff --git a/docs/src/content/docs/assets/paged_attention_comparison.png b/docs/src/content/docs/assets/paged_attention_comparison.png new file mode 100644 index 0000000000..c68b385199 Binary files /dev/null and b/docs/src/content/docs/assets/paged_attention_comparison.png differ diff --git a/docs/src/content/docs/assets/pd_architecture.jpg b/docs/src/content/docs/assets/pd_architecture.jpg new file mode 100644 index 0000000000..94678bd7ee Binary files /dev/null and b/docs/src/content/docs/assets/pd_architecture.jpg differ diff --git a/docs/src/content/docs/assets/piecewise_graph_diagram.svg b/docs/src/content/docs/assets/piecewise_graph_diagram.svg new file mode 100644 index 0000000000..c45d2c28e6 --- /dev/null +++ b/docs/src/content/docs/assets/piecewise_graph_diagram.svg @@ -0,0 +1,5558 @@ + + + + + + + + 2026-03-18T18:26:33.087555 + image/svg+xml + + + Matplotlib v3.10.8, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/src/content/docs/assets/service_arch.png b/docs/src/content/docs/assets/service_arch.png new file mode 100644 index 0000000000..d4294d8a20 Binary files /dev/null and b/docs/src/content/docs/assets/service_arch.png differ diff --git a/docs/src/content/docs/assets/shared_vmm_mapping_diagram.svg b/docs/src/content/docs/assets/shared_vmm_mapping_diagram.svg new file mode 100644 index 0000000000..2e96506212 --- /dev/null +++ b/docs/src/content/docs/assets/shared_vmm_mapping_diagram.svg @@ -0,0 +1,2474 @@ + + + + + + + + 2026-03-17T19:10:49.051734 + image/svg+xml + + + Matplotlib v3.9.4, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/src/content/docs/assets/wechat_qrcode.png b/docs/src/content/docs/assets/wechat_qrcode.png new file mode 100644 index 0000000000..1ac2ac55c0 Binary files /dev/null and b/docs/src/content/docs/assets/wechat_qrcode.png differ diff --git a/docs/src/content/docs/assets/xattention_kv_layout.png b/docs/src/content/docs/assets/xattention_kv_layout.png new file mode 100644 index 0000000000..ba797ab165 Binary files /dev/null and b/docs/src/content/docs/assets/xattention_kv_layout.png differ diff --git a/docs/src/content/docs/assets/xattention_kv_layout_en.svg b/docs/src/content/docs/assets/xattention_kv_layout_en.svg new file mode 100644 index 0000000000..3afcb4e5f7 --- /dev/null +++ b/docs/src/content/docs/assets/xattention_kv_layout_en.svg @@ -0,0 +1,82 @@ + + + + + + + + Separated KV Cache: Shared KV and Unshared KV + Shared Cache + + + + + one physical copy for shared prefix + + Unshared Cache Across Decode Rounds + + + + + + Round 0 + + + + + + + Round 1 + + + + + + + Round 2 + + + + decode + decode + + Beam Expansion Tree + + + + + + + + + + a + b + c + d + + + 0 + 1 + 2 + 3 + 4 + + + + + + + Shared KV is kept once; unshared KV grows with decode rounds and beam choices. + diff --git a/docs/src/content/docs/assets/xattention_three_stage_pipeline.png b/docs/src/content/docs/assets/xattention_three_stage_pipeline.png new file mode 100644 index 0000000000..8c6890b694 Binary files /dev/null and b/docs/src/content/docs/assets/xattention_three_stage_pipeline.png differ diff --git a/docs/src/content/docs/assets/xattention_three_stage_pipeline_en.svg b/docs/src/content/docs/assets/xattention_three_stage_pipeline_en.svg new file mode 100644 index 0000000000..b49e313570 --- /dev/null +++ b/docs/src/content/docs/assets/xattention_three_stage_pipeline_en.svg @@ -0,0 +1,55 @@ + + + xAttention Three-stage Execution Pipeline + + + Shared Stage + + MCU + BatchMatmul + BatchMatmul + BatchMatmul + BatchMatmul + + VCU + Softmax + Softmax + + + Unshared Stage + + MCU + BatchMatmul + BatchMatmul + BatchMatmul + BatchMatmul + + VCU + Softmax + Softmax + + + Merging Stage + + MCU + Post-processing + Post-processing + + VCU + OnlineSoftmax + OnlineSoftmax + OnlineSoftmax + + Shared and unshared attention are computed separately, then merged with OnlineSoftmax. + diff --git a/docs/src/content/docs/assets/xllm-ai-coding-workflow-en.png b/docs/src/content/docs/assets/xllm-ai-coding-workflow-en.png new file mode 100644 index 0000000000..8d38968986 Binary files /dev/null and b/docs/src/content/docs/assets/xllm-ai-coding-workflow-en.png differ diff --git a/docs/src/content/docs/assets/xllm-ai-coding-workflow-zh.png b/docs/src/content/docs/assets/xllm-ai-coding-workflow-zh.png new file mode 100644 index 0000000000..2581a7e5f6 Binary files /dev/null and b/docs/src/content/docs/assets/xllm-ai-coding-workflow-zh.png differ diff --git a/docs/src/content/docs/assets/xllm_arch.png b/docs/src/content/docs/assets/xllm_arch.png new file mode 100644 index 0000000000..36594f7a43 Binary files /dev/null and b/docs/src/content/docs/assets/xllm_arch.png differ diff --git a/docs/src/content/docs/en/cli_reference.md b/docs/src/content/docs/en/cli_reference.md new file mode 100644 index 0000000000..c3283e0c11 --- /dev/null +++ b/docs/src/content/docs/en/cli_reference.md @@ -0,0 +1,285 @@ +--- +title: "Service Startup Parameters" +sidebar: + order: 100 +--- + +xLLM uses gflags to manage service startup parameters. `--model ` is the only required flag. When `--config_json_file` is used, values in the JSON file override command-line flag values. The tables below are grouped by the Config classes in `/xllm/core/framework/config`, with one Config per section. The `ConfigJsonUtils` section contains the common JSON config-file flags. + +## ConfigJsonUtils + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `config_json_file` | `string` | `""` | Path to a JSON config file. Values in the file override command-line flag values. | +| `enable_dump_config_json` | `bool` | `false` | Whether to dump the resolved startup config as JSON. | +| `dump_config_json_file` | `string` | `"xllm_config.json"` | Path to write the resolved startup config as JSON. Used only when `enable_dump_config_json=true`. | + +## ServiceConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `host` | `string` | `""` | Host name or IP for the bRPC server. | +| `port` | `int32` | `8010` | Port for the bRPC server. | +| `rpc_idle_timeout_s` | `int32` | `-1` | Close the connection when there are no read/write operations during the last `rpc_idle_timeout_s` seconds. `-1` waits indefinitely. | +| `rpc_channel_timeout_ms` | `int32` | `-1` | Maximum bRPC Channel duration in milliseconds. `-1` waits indefinitely. | +| `max_reconnect_count` | `int32` | `40` | Maximum number of reconnect attempts from a worker to a server. | +| `num_threads` | `int32` | `8` | Number of threads used to process requests. | +| `max_concurrent_requests` | `int32` | `200` | Maximum number of concurrent requests the xLLM instance can handle. Set to `0` for no limit. | +| `num_request_handling_threads` | `int32` | `4` | Number of threads for handling input requests. | +| `num_response_handling_threads` | `int32` | `4` | Number of threads for handling responses. | +| `health_check_interval_ms` | `int32` | `3000` | Worker health-check interval in milliseconds. | + +## ModelConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `model_id` | `string` | `""` | Hugging Face model name, not a path. | +| `model` | `string` | `""` | Hugging Face model name or model path. | +| `backend` | `string` | `""` | Backend model type: `llm` for text-only models, `vlm` for multimodal models, or `dit` for diffusion models. | +| `task` | `string` | `"generate"` | Model task, for example `generate`, `embed`, or `mm_embed`. | +| `devices` | `string` | `""` | Deprecated; use `device_id` instead. Devices used by the current process, for example `npu:0` or `npu:0,npu:1`. | +| `device_id` | `int32` | `-1` | Device id to run the model on, for example `0`. | +| `limit_image_per_prompt` | `int32` | `8` | Maximum number of images per prompt. Only applies to multimodal models. | +| `max_encoder_cache_size` | `int64` | `0` | Maximum GPU/NPU memory size in MB for the encoder cache per worker. `0` disables the encoder cache. | +| `reasoning_parser` | `string` | `""` | Reasoning parser, for example `auto`, `glm45`, `glm47`, `glm5`, `qwen3`, `qwen35`, or `deepseek-r1`. | +| `tool_call_parser` | `string` | `""` | Tool-call parser, for example `auto`, `qwen25`, `qwen3`, `qwen35`, `qwen3_coder`, `kimi_k2`, `deepseekv3`, `deepseekv32`, `deepseekv4`, `glm45`, `glm47`, or `glm5`. | +| `enable_qwen3_reranker` | `bool` | `false` | Whether to enable the Qwen3 reranker. | +| `flashinfer_workspace_buffer_size` | `int32` | `134217728` | Reserved FlashInfer workspace buffer for intermediate attention results in split-k attention. Default is 128 MiB. | +| `enable_return_mm_full_embeddings` | `bool` | `false` | Whether VLM models return ViT embeddings and sequence embeddings. | +| `mm_download_headers` | `string` | `""` | Service-level default HTTP headers for multimodal downloads, as a JSON object. Per-request headers take precedence. Example: `{"Authorization":"Bearer xxx"}`. | +| `use_audio_in_video` | `bool` | `false` | Whether to decode both audio and video when the input is a video. | +| `use_cpp_chat_template` | `bool` | `true` | Use native C++ chat templates for supported models, for example `deepseek_v32` and `deepseek_v4`. Set to `false` to fall back to Jinja for debugging. | + +## LoadConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `enable_manual_loader` | `bool` | `false` | Pin decoder layer weights to host memory and use async H2D transfer. Required by `enable_rolling_load`; also implied by `enable_xtensor`. | +| `enable_rolling_load` | `bool` | `false` | Enable rolling weight load: keep only N decoder layer weight slots in HBM and stream-load each layer just in time. Requires `enable_manual_loader=true`. NPU only. | +| `rolling_load_num_cached_layers` | `int32` | `2` | Number of decoder layer weight slots to keep in HBM when `enable_rolling_load=true`. | +| `rolling_load_num_rolling_slots` | `int32` | `-1` | Number of rolling slots used by decoder rolling load. Fixed slots are `rolling_load_num_cached_layers - rolling_load_num_rolling_slots`. `-1` means auto, `min(2, preload_count)`. Must be in `[-1, rolling_load_num_cached_layers]`. | +| `enable_prefetch_weight` | `bool` | `false` | Whether to enable weight prefetching. Only applies to Qwen3-dense models. The default gateup weight prefetch ratio is 40%; adjust with `PREFETCH_COEFFOCIENT`. | + +## KVCacheConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `block_size` | `int32` | `128` | Number of slots per KV Cache block. | +| `max_cache_size` | `int64` | `0` | Maximum GPU memory size for KV Cache. `0` means calculated from available memory. | +| `max_memory_utilization` | `double` | `0.8` | Fraction of GPU memory used for model inference, including model weights and KV Cache. | +| `kv_cache_dtype` | `string` | `"auto"` | KV Cache dtype for quantization. `auto` aligns with model dtype and disables quantization. `int8` enables INT8 quantization and is only supported on the MLU backend. | +| `enable_prefix_cache` | `bool` | `true` | Whether to enable prefix cache in the block manager. See [Prefix Cache](/en/features/prefix_cache/). | +| `enable_in_batch_prefix_cache` | `bool` | `false` | Whether to cache admitted prefill full blocks into the prefix cache so that later requests in the same batch can share them. | +| `max_linear_state_cache_slots` | `int64` | `0` | Maximum number of active linear-attention state cache slots. `0` derives an automatic capacity from the available KV Cache budget. | +| `xxh3_128bits_seed` | `uint32` | `1024` | Default XXH3 128-bit hash seed. | +| `enable_xtensor` | `bool` | `false` | Whether to enable XTensor for model weights with the physical page pool. | +| `phy_page_granularity_size` | `int64` | `2097152` | Granularity size of one physical page in bytes, default 2 MiB, for continuous KV Cache. | + +## KVCacheStoreConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `prefetch_timeout` | `uint32` | `0` | Timeout for prefetching from KV Cache Store. | +| `prefetch_batch_size` | `uint32` | `2` | Copy batch size for prefetching from KV Cache Store. | +| `layers_wise_copy_batchs` | `uint32` | `4` | Number of batches for layer-wise H2D copy. | +| `host_blocks_factor` | `double` | `0.0` | Host block factor, for example `host block num = host_blocks_factor * hbm block num`. | +| `enable_kvcache_store` | `bool` | `false` | Whether to enable KV Cache Store. | +| `store_protocol` | `string` | `"tcp"` | KV Cache Store protocol, for example `tcp` or `rdma`. | +| `store_master_server_address` | `string` | `""` | Address information of the Store master service. | +| `store_metadata_server` | `string` | `""` | Address of the KV Cache Store metadata service. | +| `store_local_hostname` | `string` | `""` | Local host name of the KV Cache Store client. | +| `enable_control_h2d_block_num` | `bool` | `false` | Whether to control the number of H2D copy blocks. | + +## BeamSearchConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `enable_beam_search_kernel` | `bool` | `false` | Whether to enable the beam search kernel. | +| `beam_width` | `int32` | `1` | Beam width for beam search. | +| `enable_block_copy_kernel` | `bool` | `true` (NPU/CUDA); `false` (other backends) | Whether to use the block copy kernel on supported backends. | +| `enable_topk_sorted` | `bool` | `true` | Whether to enable sorted top-k output. | + +## SchedulerConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `max_tokens_per_batch` | `int32` | `10240` | Maximum number of tokens per batch. | +| `max_seqs_per_batch` | `int32` | `1024` | Maximum number of sequences per batch. | +| `enable_schedule_overlap` | `bool` | `false` | Whether to enable schedule overlap, also known as asynchronous scheduling. See [Async Scheduling](/en/features/async_schedule/). | +| `prefill_scheduling_memory_usage_threshold` | `double` | `0.95` | Memory usage threshold during prefill scheduling. | +| `enable_chunked_prefill` | `bool` | `true` | Whether to enable chunked prefill. | +| `max_tokens_per_chunk_for_prefill` | `int32` | `-1` | Maximum number of tokens per chunk in the prefill stage. `-1` uses the default policy. | +| `chunked_match_frequency` | `int32` | `2` | Sequence prefix-cache match frequency. | +| `use_zero_evict` | `bool` | `false` | Whether to use ZeroEvictionScheduler. See [Zero Evict Scheduler](/en/features/zero_evict_scheduler/). | +| `max_decode_token_per_sequence` | `int32` | `256` | Maximum decode tokens per sequence for ZeroEvictionScheduler. | +| `priority_strategy` | `string` | `"fcfs"` | Request priority strategy, for example `fcfs`, `priority`, or `deadline`. | +| `use_mix_scheduler` | `bool` | `false` | Whether to use MixScheduler to handle prefill and decode uniformly. | +| `enable_online_preempt_offline` | `bool` | `true` | Whether online requests can preempt offline requests. | +| `aggressive_coeff` | `double` | `1.0` | Aggressive coefficient for MixScheduler urgency judgment. | +| `starve_threshold` | `double` | `1.0` | Starvation threshold coefficient for MixScheduler. | +| `enable_starve_prevent` | `bool` | `true` | Whether to enable anti-starvation behavior in MixScheduler. | + +## ParallelConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `dp_size` | `int32` | `1` | Data parallel size for MLA attention. | +| `ep_size` | `int32` | `1` | Expert parallel size for MoE models. | +| `cp_size` | `int32` | `1` | Context parallel size for DSA attention. | +| `kv_split_size` | `int32` | `1` | KV Cache split width. `0` falls back to `cp_size` (legacy). `1` disables KV split: each CP rank stores the full KV and skips the prefix AllGather. Other K values (K divides `cp_size`) shard KV across K ranks while token-CP still uses `cp_size`. | +| `tp_size` | `int64` | `1` | Tensor parallelism size. Only used for DiT models. | +| `sp_size` | `int64` | `1` | Sequence parallelism size. Only used for DiT models. | +| `cfg_size` | `int64` | `1` | Classifier-free guidance parallelism size. Only used for DiT models. | +| `vae_size` | `int64` | `1` | VAE patch parallelism size. Only used for DiT models. | +| `communication_backend` | `string` | `"hccl"` | NPU communication backend, for example `lccl` or `hccl`. Uses `hccl` when DP is enabled. | +| `enable_prefill_sp` | `bool` | `false` | Whether to enable prefill-only sequence parallel. Supports `enable_chunked_prefill=true`, but only for prefill-only batches (`PREFILL` / `CHUNKED_PREFILL`). | +| `enable_mm_encoder_dp` | `bool` | `false` | Whether to enable encoder data parallelism for multimodal models. | +| `enable_multi_stream_parallel` | `bool` | `false` | Whether to enable computation/communication parallelism with two streams and two micro batches in the prefill stage. See [Multi-Stream Parallelism](/en/features/multi_streams/). | +| `micro_batch_num` | `int32` | `1` | Number of micro batches used for multi-stream parallelism. | +| `enable_dp_balance` | `bool` | `false` | Whether to enable DP load balancing. If true, sequences within a single DP batch are shuffled. | + +## EPLBConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `enable_eplb` | `bool` | `false` | Whether to enable expert parallel load balance. See [EPLB](/en/features/eplb/). | +| `redundant_experts_num` | `int32` | `1` | Number of redundant experts per device. | +| `eplb_update_interval` | `int64` | `1000` | EPLB update interval. | +| `eplb_update_threshold` | `double` | `0.8` | EPLB update threshold. | +| `expert_parallel_degree` | `int32` | `0` | Expert parallel degree. | +| `rank_tablefile` | `string` | `""` | ATB HCCL rank table file. | + +## DistributedConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `master_node_addr` | `string` | `"127.0.0.1:19888"` | Master address for multi-node distributed serving, for example `10.18.1.1:9999`. | +| `xtensor_master_node_addr` | `string` | `"127.0.0.1:19889"` | Master address for the XTensor distributed service, for example `10.18.1.1:9999`. | +| `nnodes` | `int32` | `1` | Number of multi-node nodes. | +| `node_rank` | `int32` | `0` | Rank of the current node. | +| `etcd_addr` | `string` | `""` | etcd address used to save instance metadata. | +| `etcd_namespace` | `string` | `""` | Optional etcd namespace prefix for all xLLM keys, for example `prod-a`. | +| `enable_service_routing` | `bool` | `false` | Whether to enable xLLM service routing. | +| `heart_beat_interval` | `double` | `0.5` | Heartbeat interval. | +| `etcd_ttl` | `int32` | `3` | Time to live for etcd keys. | + +## DisaggPDConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `enable_disagg_pd` | `bool` | `false` | Whether to enable disaggregated prefill and decode execution. See [P-D Separation](/en/features/disagg_pd/). | +| `enable_pd_ooc` | `bool` | `false` | Whether to enable online-offline co-location in disaggregated PD mode. | +| `disagg_pd_port` | `int32` | `7777` | Listening port for the disaggregated PD bRPC server. | +| `instance_role` | `string` | `"DEFAULT"` | Instance role, for example `DEFAULT`, `PREFILL`, `DECODE`, or `MIX`. | +| `kv_cache_transfer_type` | `string` | `"LlmDataDist"` | KV Cache transfer type, for example `LlmDataDist`, `Mooncake`, or `HCCL`. | +| `kv_cache_transfer_mode` | `string` | `"PUSH"` | KV Cache transfer mode, for example `PUSH` or `PULL`. | +| `transfer_listen_port` | `int32` | `26000` | Listening port for KV Cache Transfer. | +| `kv_push_dst_rotate` | `bool` | `false` | Rotate the destination-worker traversal order in `push_kv_blocks` per KV-split rank to spread incast across decode workers. | + +## SpeculativeConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `draft_model` | `string` | `""` | Draft model path. See [MTP](/en/features/mtp/) for MTP usage. | +| `draft_devices` | `string` | `""` | Devices used by the draft model, for example `npu:0` or `npu:0,npu:1`. If omitted, uses the target model devices when speculative decoding is enabled. | +| `num_speculative_tokens` | `int32` | `0` | Number of speculative tokens generated per speculative decoding step. | +| `speculative_algorithm` | `string` | `"MTP"` | Speculative decoding algorithm. Supported values: `MTP`, `Eagle3`, `Suffix`. | +| `speculative_suffix_cache_max_depth` | `int32` | `64` | Maximum suffix-tree depth for suffix speculative decoding. | +| `speculative_suffix_max_spec_factor` | `double` | `1.0` | Maximum suffix speculation token factor relative to match length. | +| `speculative_suffix_max_spec_offset` | `double` | `0.0` | Maximum additive token offset for suffix speculation. | +| `speculative_suffix_min_token_prob` | `double` | `0.1` | Minimum token probability used in suffix speculation. | +| `speculative_suffix_max_cached_requests` | `int32` | `-1` | Maximum number of globally cached requests for suffix speculation. `-1` means unlimited; `0` disables it. | +| `speculative_suffix_use_tree_spec` | `bool` | `false` | Whether to use tree-based suffix speculation instead of path speculation. | +| `enable_opt_validate_probs` | `bool` | `false` | Whether validation uses selected-only `draft_probs [B,S]` directly. If false, selected-only cache values are restored to dense `[B,S,V]`. | +| `enable_atb_spec_kernel` | `bool` | `false` | Whether to use the ATB speculative kernel. | + +## ProfileConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `enable_profile_step_time` | `bool` | `false` | Whether to enable step-time profiling. | +| `enable_profile_token_budget` | `bool` | `false` | Whether to enable token-budget profiling. | +| `enable_latency_aware_schedule` | `bool` | `false` | Whether to use predicted latency for latency-aware scheduling. | +| `profile_max_prompt_length` | `int32` | `2048` | Maximum prompt length used for profiling. | +| `max_global_ttft_ms` | `int32` | `std::numeric_limits::max()` | Global TTFT threshold in milliseconds. | +| `max_global_tpot_ms` | `int32` | `std::numeric_limits::max()` | Global TPOT threshold in milliseconds. | +| `enable_profile_kv_blocks` | `bool` | `true` | Whether to generate KV Cache blocks for profiling. | +| `disable_ttft_profiling` | `bool` | `false` | Whether to disable TTFT profiling. | +| `enable_forward_interruption` | `bool` | `false` | Whether to enable forward interruption. | +| `enable_online_profile` | `bool` | `false` | Whether to enable the online timeline profiling endpoints (`/start_profile` and `/stop_profile`). CUDA only for now; pair with launching the server under `nsys --capture-range=cudaProfilerApi`. | +| `profile_backend` | `string` | `"torch"` | Online profiling backend. `torch` records CPU+CUDA activities in-process and writes a Chrome trace on `/stop_profile`, no external profiler needed. `cuda` only toggles the CUDA profiler capture range and requires launching under `nsys --capture-range=cudaProfilerApi`. | +| `profile_dir` | `string` | `""` | Directory the `torch` online profiling backend writes timeline traces to. Empty means the current working directory. | + +## ExecutionConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `enable_graph` | `bool` | `false` | Whether to enable graph execution for the decode phase to reduce kernel-launch overhead and device idle time. Supports CUDA Graph, ACL Graph (NPU), MLU Graph, and DCU Graph. See [Graph Mode](/en/features/graph_mode/). | +| `enable_graph_double_buffer` | `bool` | `true` | Whether to enable double-buffered ACL graph persistent params and graph instances for NPU schedule-overlap decode. | +| `enable_graph_mode_decode_no_padding` | `bool` | `false` | Whether decode graph capture uses the actual `num_tokens` instead of a padded shape. | +| `enable_prefill_piecewise_graph` | `bool` | `false` | Whether to enable piecewise CUDA graph for the prefill phase. Attention runs in eager mode while other operations are captured in CUDA graphs. | +| `enable_graph_vmm_pool` | `bool` | `true` | Whether to enable a VMM-backed CUDA graph memory pool for multi-shape graph memory reuse. | +| `max_tokens_for_graph_mode` | `int32` | `2048` | Maximum number of tokens for graph execution. `0` means no limit. | +| `acl_graph_decode_batch_size_limit` | `int32` | `16` | Decode batch size threshold for ACL graph on NPU. When the actual decode batch size exceeds this value, ACL graph decode falls back to eager mode to avoid OOM. | +| `enable_shm` | `bool` | `false` | Whether to enable shared memory for model execution. | +| `use_contiguous_input_buffer` | `bool` | `true` | Whether to use a contiguous device input buffer for model execution. | +| `input_shm_size` | `uint64` | `1024` | Input shared-memory size. Default is 1GB. | +| `output_shm_size` | `uint64` | `128` | Output shared-memory size. Default is 128MB. | +| `random_seed` | `int32` | `-1` | Random seed for the random number generator. `-1` means no fixed seed. | + +## KernelConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `enable_customize_mla_kernel` | `bool` | `false` | Whether to enable the customized MLA kernel. | +| `npu_kernel_backend` | `string` | `"AUTO"` | NPU kernel backend. Supported values: `AUTO`, `ATB`, `TORCH`. | +| `enable_intralayer_addnorm` | `bool` | `false` | Whether to enable fused intralayer addnorm ops. | +| `enable_fused_mc2` | `int32` | `-1` | Fused MC2 mode for NPU. `-1` uses the auto default, `0` disables fused MC2, positive values enable dense matmul-allreduce, `1` uses DispatchFFNCombine for MoE, `2` uses DispatchGmmCombineDecode for MoE. | +| `enable_interlayer_addnorm` | `bool` | `false` | Whether to enable fused interlayer addnorm ops. | +| `enable_split_rmsnorm_rope` | `bool` | `false` | Whether to enable fused split rmsnorm rope ops. | +| `enable_aclnn_matmul` | `bool` | `false` | Whether to enable the ACLNN matmul backend for supported NPU ATB layers. | +| `enable_aclnn_swiglu` | `bool` | `false` | Whether to enable the ACLNN SwiGLU backend for supported NPU ATB layers. | + +## DiTConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `max_requests_per_batch` | `int32` | `1` | Maximum number of requests per batch. | +| `dit_cache_policy` | `string` | `"TaylorSeer"` | DiT cache policy, for example `None`, `FBCache`, `TaylorSeer`, `FBCacheTaylorSeer`, or `ResidualCache`. | +| `dit_cache_warmup_steps` | `int64` | `0` | Number of warmup steps. | +| `dit_cache_n_derivatives` | `int64` | `3` | Number of derivatives to use in TaylorSeer. | +| `dit_cache_skip_interval_steps` | `int64` | `3` | Interval steps to skip for derivative calculation. | +| `dit_cache_residual_diff_threshold` | `double` | `0.09` | Residual difference threshold for cache reuse. | +| `dit_cache_start_steps` | `int64` | `5` | Number of steps to skip at the start. | +| `dit_cache_end_steps` | `int64` | `5` | Number of steps to skip at the end. | +| `dit_cache_start_blocks` | `int64` | `5` | Number of blocks to skip at the start. | +| `dit_cache_end_blocks` | `int64` | `5` | Number of blocks to skip at the end. | +| `dit_sp_communication_overlap` | `bool` | `true` | Whether to overlap communication and computation for sequence parallelism. | +| `dit_debug_print` | `bool` | `false` | Whether to print debug information for DiT models. | +| `dit_generation_image_area_max` | `int64` | `0` | Maximum allowed image area (`width * height`) for image generation requests. `0` means no limit. | +| `dit_enable_vae_tiling` | `bool` | `false` | Whether to enable VAE tiling. Currently only supported for `qwen-image-edit-plus`. | +| `dit_vae_image_size` | `int64` | `1048576` | Qwen-Image-Edit-Plus VAE image size used to calculate dimensions. | +| `dit_sparse_attention_enabled` | `bool` | `false` | Whether to enable block-wise sparse attention / RainFusion for WAN. | +| `dit_sparse_attention_sparsity` | `double` | `0.5` | Sparse attention sparsity ratio in `[0.0, 1.0)`. `0.0` means dense attention, `0.5` drops 50 percent of blocks. | +| `dit_sparse_attention_pool_size` | `int64` | `128` | Sparse attention pooling window size for block-wise mask generation. | +| `dit_sparse_attention_sparse_start_step` | `int64` | `0` | Sparse attention step index to start sparse attention. Steps before this use dense attention. | +| `dit_sparse_attention_version` | `string` | `"rain_fusion"` | Sparse attention version: `rain_fusion` (frame-pairing + `aclnnRainFusionAttention`) or `sparse_attention` (block-decompose + `aclnnBlockSparseAttention`). | +| `dit_sparse_attention_mask_refresh_steps` | `int64` | `1` | Recompute the block sparse mask every N diffusion steps. `1` means every step, higher values reuse the mask longer. | + +## RecConfig + +| Parameter | Type | Default | Description | +|:----------|:-----|:--------|:------------| +| `enable_rec_fast_sampler` | `bool` | `true` | Whether to enable the RecSampler fast sampling path for Rec pipelines. | +| `enable_rec_prefill_only` | `bool` | `false` | Whether to enable Rec prefill-only mode without decoder self-attention block allocation. | +| `enable_xattention_one_stage` | `bool` | `false` | Whether to force xattention one-stage decode for Rec multi-round mode. | +| `max_decode_rounds` | `int32` | `0` | Maximum number of decode rounds for multi-step decoding. `0` means disabled. | +| `enable_constrained_decoding` | `bool` | `false` | Whether to enable constrained decoding with predefined rules for output format or structure. | +| `output_rec_logprobs` | `bool` | `false` | Whether to output Rec multi-round token-aligned logprobs. Missing per-token logprobs are filled with the final beam logprob. | +| `enable_convert_tokens_to_item` | `bool` | `false` | Whether to convert token IDs to item IDs in REC/OneRec responses. | +| `enable_output_sku_logprobs` | `bool` | `false` | Whether to output REC/OneRec token-aligned logprobs tensors. | +| `enable_extended_item_info` | `bool` | `false` | Whether to parse and output REC extended item info tensors. | +| `each_conversion_threshold` | `int32` | `50` | Maximum number of items emitted for each REC token triplet. | +| `total_conversion_threshold` | `int32` | `1000` | Maximum total number of items emitted in one REC response. | +| `request_queue_size` | `int32` | `100000` | Scheduler request queue size. | +| `rec_worker_max_concurrency` | `uint32` | `1` | Concurrency for Rec worker parallel execution. Values less than or equal to `1` disable concurrent Rec workers. | diff --git a/docs/src/content/docs/en/cookbook.md b/docs/src/content/docs/en/cookbook.md new file mode 100644 index 0000000000..6d9b7b9d50 --- /dev/null +++ b/docs/src/content/docs/en/cookbook.md @@ -0,0 +1,13 @@ +--- +title: "CookBook" +description: "Practical xLLM recipes and deployment examples" +--- + +This section collects practical xLLM recipes, deployment examples, and troubleshooting notes for common inference scenarios. + +Content will be added as the recipes are organized. + +## Sections + +- [Autoregressive Models](/en/cookbook/autoregressive_models/) +- [Diffusion Models](/en/cookbook/diffusion_models/) diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models.md b/docs/src/content/docs/en/cookbook/autoregressive_models.md new file mode 100644 index 0000000000..6706a6cfa8 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models.md @@ -0,0 +1,16 @@ +--- +title: "Autoregressive Models" +description: "CookBook recipes for autoregressive model inference with xLLM" +--- + +This section collects CookBook recipes for serving autoregressive models with xLLM. + +Content will be added as the recipes are organized. + +## Model Families + +- [Qwen](/en/cookbook/autoregressive_models/qwen/qwen3_5/) +- [DeepSeek](/en/cookbook/autoregressive_models/deepseek/deepseek_v4/) +- [GLM](/en/cookbook/autoregressive_models/glm/glm_5/) +- [Kimi](/en/cookbook/autoregressive_models/kimi/kimi2/) +- [MinMax](/en/cookbook/autoregressive_models/minmax/minmax_m2_7/) diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_r1.md b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_r1.md new file mode 100644 index 0000000000..5d2c23f601 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_r1.md @@ -0,0 +1,8 @@ +--- +title: "DeepSeek-R1" +description: "CookBook recipe placeholder for DeepSeek-R1 autoregressive model inference" +--- + +This section will collect DeepSeek-R1 autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v3.md b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v3.md new file mode 100644 index 0000000000..ad28674692 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v3.md @@ -0,0 +1,8 @@ +--- +title: "DeepSeek-V3" +description: "CookBook recipe placeholder for DeepSeek-V3 autoregressive model inference" +--- + +This section will collect DeepSeek-V3 autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v3_1.md b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v3_1.md new file mode 100644 index 0000000000..2c1076767f --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v3_1.md @@ -0,0 +1,8 @@ +--- +title: "DeepSeek-V3.1" +description: "CookBook recipe placeholder for DeepSeek-V3.1 autoregressive model inference" +--- + +This section will collect DeepSeek-V3.1 autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v3_2.md b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v3_2.md new file mode 100644 index 0000000000..864b46020d --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v3_2.md @@ -0,0 +1,8 @@ +--- +title: "DeepSeek-V3.2" +description: "CookBook recipe placeholder for DeepSeek-V3.2 autoregressive model inference" +--- + +This section will collect DeepSeek-V3.2 autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md new file mode 100644 index 0000000000..f68b06b4af --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md @@ -0,0 +1,269 @@ +--- +title: "DeepSeek-V4" +description: "DeepSeek-V4 inference guide with xLLM on Ascend A3 devices" +--- + +# Inference with xLLM on Ascend A3 Devices + +Source code: https://github.com/jd-opensource/xllm + +China mirror: https://gitcode.com/xLLM-AI/xllm + +Weight Download +Flash weights: +https://modelers.cn/models/Eco-Tech/DeepSeek-V4-Flash-w8a8-mtp + +Pro weights: +https://modelers.cn/models/Eco-Tech/DeepSeek-V4-Pro-w4a8-mtp + + +## 1. Pull the Docker Image + +First, pull the xLLM-provided image: + +```bash +# A2 x86 +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-x86-cann9-20260605 +# A2 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-arm-cann9-20260605 +# A3 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-cann9-20260605 +``` + +Then create the container: + +```bash +sudo docker run -it --ipc=host -u 0 --privileged --name mydocker --network=host \ + -v /var/queue_schedule:/var/queue_schedule \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ + -v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \ + -v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \ + -v /var/log/npu/slog/:/var/log/npu/slog \ + -v ~/.ssh:/root/.ssh \ + -v /var/log/npu/profiling/:/var/log/npu/profiling \ + -v /var/log/npu/dump/:/var/log/npu/dump \ + -v /runtime/:/runtime/ -v /etc/hccn.conf:/etc/hccn.conf \ + -v /export/home:/export/home \ + -v /home/:/home/ \ + -w /export/home \ + quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-cann9-20260605 +``` + +## 2. Clone Source Code and Build + +Clone the official repository and module dependencies: + +```bash +git clone https://github.com/jd-opensource/xllm +cd xllm +git submodule update --init --recursive +``` + +Install dependencies: + +```bash +pip install --upgrade pre-commit +``` + +Build the project; the executable `build/xllm/core/server/xllm` will be generated under `build/`: + +```bash +python setup.py build --device npu +``` + +## 3. Launch the Model + +### If restarting after a machine reboot, initialize the device first + +> If not executed and the NPU is not initialized, the xllm process may fail to start + +```bash +python -c "import torch_npu +for i in range(16):torch_npu.npu.set_device(i)" +``` + +### Export MTP weights + +```bash +python tools/export_mtp.py --input-dir ${W4A8/W8A8 weights directory} --output-dir ${Exported MTP weights directory} +``` + +### Environment variables + +```bash +##### 1. Configure dependency path environment variables + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source ${ASCEND_TOOLKIT_HOME}/opp/vendors/custom_xllm_math/bin/set_env.bash + +##### 2. Configure logging environment variables +rm -rf /root/ascend/log/ +rm -rf core.* + +##### 3. Configure performance and communication environment variables +export HCCL_IF_BASE_PORT=43432 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export NPU_MEMORY_FRACTION=0.96 +export ATB_WORKSPACE_MEM_ALLOC_ALG_TYPE=3 +export ATB_WORKSPACE_MEM_ALLOC_GLOBAL=1 +export ATB_LAYER_INTERNAL_TENSOR_REUSE=1 +export ATB_CONTEXT_WORKSPACE_SIZE=0 +export OMP_NUM_THREADS=12 +export ALLOW_INTERNAL_FORMAT=1 + +``` + +## Launch Command - Single-Node Example + +```bash +BATCH_SIZE=256 +# Maximum batch size for inference +XLLM_PATH="./myxllm/xllm/build/xllm/core/server/xllm" +# Path to the inference entry file (build artifact from the previous step) +MODEL_PATH=/path/to/dsv4 +# Model path +DRAFT_MODEL_PATH=/path/to/dsv4_mtp +# Exported MTP weights path + +MASTER_NODE_ADDR="11.87.49.110:10015" +LOCAL_HOST="11.87.49.110" +# Service Port +START_PORT=18994 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=8 + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + nohup $XLLM_PATH -model-id ds \ + --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + --max_memory_utilization=0.9 \ + --max_tokens_per_batch=2048 \ + --max_seqs_per_batch=32 \ + --block_size=128 \ + --communication_backend="hccl" \ + --tool_call_parser=deepseekv4 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=true \ + --enable_schedule_overlap=true \ + --enable_graph=true \ + --npu_kernel_backend=TORCH \ + --ep_size=8 \ + --dp_size=2 \ + > $LOG_FILE 2>&1 & +done + + # Variables required when MTP is enabled + # --draft_model=$DRAFT_MODEL_PATH \ + # --draft_devices="npu:$DEVICE" \ + # --num_speculative_tokens=1 \ + +# numactl -C xxxxx NUMA core binding (query with: npu-smi info -t topo) +#--max_memory_utilization Max memory usage ratio per NPU card +#--max_tokens_per_batch Max tokens per batch (mainly limits prefill) +#--max_seqs_per_batch Max sequences per batch (mainly limits decode) +#--communication_backend Communication backend (hccl / lccl, hccl recommended here) +#--enable_schedule_overlap Enable async scheduling +#--enable_prefix_cache Enable prefix cache +#--enable_chunked_prefill Enable chunked prefill +#--enable_graph Enable aclgraph +#--draft_model MTP - MTP weights path +#--draft_devices MTP - MTP inference device (same as main model) +#--num_speculative_tokens MTP - Number of speculative tokens +``` + +A log message "Brpc Server Started" indicates the service has started successfully. + +## Other Optional Environment Variables + +```bash +#Enable deterministic computation +export LCCL_DETERMINISTIC=1 +export HCCL_DETERMINISTIC=true +export ATB_MATMUL_SHUFFLE_K_ENABLE=0 + +# #Enable dynamic profiling mode +# export PROFILING_MODE=dynamic +# \rm -rf ~/dynamic_profiling_socket_* +``` + +## Launch Command - Dual-Node Example + +### Node0 (master) + +```bash +MASTER_NODE_ADDR="11.87.49.110:19990" +LOCAL_HOST="11.87.49.110" +START_PORT=15890 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=32 +LOCAL_NODES=16 +export HCCL_IF_BASE_PORT=48439 +unset HCCL_OP_EXPANSION_MODE + +for (( i=0; i<$LOCAL_NODES; i++ )); do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + nohup $XLLM_PATH \ + --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + ...... + --rank_tablefile=/yourPath/ranktable.json \ + > $LOG_FILE 2>&1 & +done +``` + +#### Node1 (worker) + +```bash +MASTER_NODE_ADDR="11.87.49.110:19990" +LOCAL_HOST="11.87.49.111" +START_PORT=15890 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=32 +LOCAL_NODES=16 +export HCCL_IF_BASE_PORT=48439 +unset HCCL_OP_EXPANSION_MODE + +for (( i=0; i<$LOCAL_NODES; i++ )); do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + nohup $XLLM_PATH \ + --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$((i + LOCAL_NODES)) \ + ...... + --rank_tablefile=/yourPath/ranktable.json \ + > $LOG_FILE 2>&1 & +done +``` + +### ranktable reference + + [A3 ranktable configuration](https://www.hiascend.com/document/detail/zh/canncommercial/900/API/hcclug/hcclug_000066.html) + + [A2 ranktable configuration](https://www.hiascend.com/document/detail/zh/canncommercial/900/API/hcclug/hcclug_000067.html) + + (Note the ranktable format differences between A3 and A2) diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_5.md b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_5.md new file mode 100644 index 0000000000..e3def96589 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_5.md @@ -0,0 +1,8 @@ +--- +title: "GLM-4.5" +description: "CookBook recipe placeholder for GLM-4.5 autoregressive model inference" +--- + +This section will collect GLM-4.5 autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_5v.md b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_5v.md new file mode 100644 index 0000000000..c66d734e25 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_5v.md @@ -0,0 +1,8 @@ +--- +title: "GLM-4.5V" +description: "CookBook recipe placeholder for GLM-4.5V autoregressive model inference" +--- + +This section will collect GLM-4.5V autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_6.md b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_6.md new file mode 100644 index 0000000000..9e471a0ae5 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_6.md @@ -0,0 +1,8 @@ +--- +title: "GLM-4.6" +description: "CookBook recipe placeholder for GLM-4.6 autoregressive model inference" +--- + +This section will collect GLM-4.6 autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_6v.md b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_6v.md new file mode 100644 index 0000000000..448f52949b --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_6v.md @@ -0,0 +1,8 @@ +--- +title: "GLM-4.6V" +description: "CookBook recipe placeholder for GLM-4.6V autoregressive model inference" +--- + +This section will collect GLM-4.6V autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_7.md b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_7.md new file mode 100644 index 0000000000..36e68144bd --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_7.md @@ -0,0 +1,8 @@ +--- +title: "GLM-4.7" +description: "CookBook recipe placeholder for GLM-4.7 autoregressive model inference" +--- + +This section will collect GLM-4.7 autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_7_flash.md b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_7_flash.md new file mode 100644 index 0000000000..9594705f57 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_4_7_flash.md @@ -0,0 +1,8 @@ +--- +title: "GLM-4.7-Flash" +description: "CookBook recipe placeholder for GLM-4.7-Flash autoregressive model inference" +--- + +This section will collect GLM-4.7-Flash autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md new file mode 100644 index 0000000000..fac8101a9d --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md @@ -0,0 +1,536 @@ +--- +title: "GLM-5 / GLM-5.1 / GLM-5.2" +sidebar: + order: 2 +--- ++ Source code: https://github.com/xLLM-AI/xllm + ++ Available in China: https://gitcode.com/xLLM-AI/xllm + ++ Weight downloads: + + [modelscope-GLM-5-W8A8](https://www.modelscope.cn/models/Eco-Tech/GLM-5-W8A8-xLLM-0403/files) + + [modelscope-GLM-5.1-W8A8](https://www.modelscope.cn/models/Eco-Tech/GLM-5.1-W8A8-xLLM/files) + + [modelscope-GLM-5.1-W4A8](https://www.modelscope.cn/models/Eco-Tech/GLM-5.1-w4a8) + + [modelscope-GLM-5.2-W8A8](https://www.modelscope.cn/models/Eco-Tech/GLM-5.2-W8A8/files) + +## 1. Pull the Image Environment + +First, download the image provided by xLLM: + +```bash +# A2 x86 +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-x86-cann9-20260605 +# A2 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-arm-cann9-20260605 +# A3 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-cann9-20260605 +``` + +**Note**: Performance stress testing has not been performed on A2 machines. + +Then create the corresponding container: + +```bash +sudo docker run -it --ipc=host -u 0 --privileged --name mydocker --network=host \ + -v /var/queue_schedule:/var/queue_schedule \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ + -v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \ + -v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \ + -v /var/log/npu/slog/:/var/log/npu/slog \ + -v ~/.ssh:/root/.ssh \ + -v /var/log/npu/profiling/:/var/log/npu/profiling \ + -v /var/log/npu/dump/:/var/log/npu/dump \ + -v /runtime/:/runtime/ -v /etc/hccn.conf:/etc/hccn.conf \ + -v /export/home:/export/home \ + -v /home/:/home/ \ + -w /export/home \ + quay.io/jd_xllm/xllm-ai:xllm-dev-hb-rc2-x86 +``` + +## 2. Pull the Source Code and Build + +Download the official repository and module dependencies: + +```bash +git clone https://github.com/xLLM-AI/xllm.git +cd xllm +git checkout release/v0.10.0 +git submodule update --init --recursive +``` + +Download and install dependencies: + +```bash +pip install --upgrade pre-commit +yum install numactl +``` + +Run the build. The executable `build/xllm/core/server/xllm` will be generated under `build/`: + +```bash +python setup.py build --device npu +``` + +## 3. Start the Model + +### If the service is being started for the first time after the machine has rebooted, run the following script first to initialize the devices + +If this is skipped and the NPU has not been initialized, the xLLM process may fail to start. + +```bash +python -c "import torch_npu +for i in range(16):torch_npu.npu.set_device(i)" +``` + +### Export MTP Weights + +```bash +python tools/export_mtp.py --input-dir ${W4A8/W8A8_WEIGHT_DIR} --output-dir ${EXPORTED_MTP_WEIGHT_DIR} +``` + +### Environment Variables + +```bash +##### 1. Configure environment variables +export LD_PRELOAD=/usr/lib64/libtcmalloc.so.4:$LD_PRELOAD +export HCCL_EXEC_TIMEOUT=300 +export HCCL_CONNECT_TIMEOUT=300 +export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_IF_BASE_PORT=2864 + +##### 2. Clean residual logs +rm -rf /root/ascend/log/ +``` + +## Startup Command - A3 Single Node - GLM-5.2-W8A8 + +```bash +XLLM_PATH="./myxllm/xllm/build/xllm/core/server/xllm" +# Path to the xLLM executable +MODEL_PATH=/path/to/GLM-5.2-W8A8/ +# Model path, using GLM-5.2-W8A8 as an example +DRAFT_MODEL_PATH=/path/to/GLM-5.2-MTP/ +# MTP weights exported in the previous step + +MASTER_NODE_ADDR="11.87.49.110:10015" +LOCAL_HOST="11.87.49.110" +# Service port +START_PORT=18994 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=16 + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + # Optional: bind CPU cores with numactl. Query NUMA affinity with: npu-smi info -t topo + #nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + nohup $XLLM_PATH \ + --model $MODEL_PATH \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + --max_memory_utilization=0.86 \ + --max_tokens_per_batch=4096 \ + --max_seqs_per_batch=16 \ + --block_size=128 \ + --enable_prefix_cache=true \ + --enable_chunked_prefill=true \ + --enable_graph=true \ + --enable_schedule_overlap=true \ + --communication_backend="hccl" \ + --graph_decode_batch_size_limit=2 \ + --draft_model=$DRAFT_MODEL_PATH \ + --draft_devices="npu:$DEVICE" \ + --num_speculative_tokens=3 \ + --ep_size=16 \ + --dp_size=2 \ + --tool_call_parser=auto \ + > $LOG_FILE 2>&1 & +done + +# --max_memory_utilization Maximum memory utilization ratio per card. +# --max_tokens_per_batch Maximum tokens per batch. Mainly limits prefill. +# --max_seqs_per_batch Maximum requests per batch. Mainly limits decode. +# --communication_backend Communication backend. Options: hccl / lccl. hccl is recommended here. +# --enable_schedule_overlap Enable async scheduling. +# --enable_prefix_cache Enable prefix cache. +# --enable_chunked_prefill Enable chunked prefill. +# --enable_graph Enable aclgraph. It requires extra memory. +# --acl_graph_decode_batch_size_limit Maximum decode batch size for graph capture. It must be <= 32 / (number of speculative tokens + 1). +# --draft_model MTP draft-model weight path. +# --draft_devices MTP inference device, the same as the main model. +# --num_speculative_tokens Number of speculative tokens predicted by MTP. +``` + +When the log contains `"Brpc Server Started"`, the service has started successfully. + +## Other Optional Environment Variables + +```bash +# Enable deterministic computation +export LCCL_DETERMINISTIC=1 +export HCCL_DETERMINISTIC=true +export ATB_MATMUL_SHUFFLE_K_ENABLE=0 + +# Enable dynamic profiling mode +export PROFILING_MODE=dynamic +\rm -rf ~/dynamic_profiling_socket_* +``` + +## Startup Command - Two-Machine Startup Example + +### Node0 (master) + +```bash +MASTER_NODE_ADDR="11.87.49.110:19990" +LOCAL_HOST="11.87.49.110" +START_PORT=15890 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=32 +LOCAL_NODES=16 +export HCCL_IF_BASE_PORT=48439 +unset HCCL_OP_EXPANSION_MODE + +for (( i=0; i<$LOCAL_NODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + --max_memory_utilization=0.85 \ + --max_tokens_per_batch=8192 \ + --max_seqs_per_batch=128 \ + --block_size=128 \ + --enable_prefix_cache=true \ + --enable_chunked_prefill=true \ + --communication_backend="hccl" \ + --enable_schedule_overlap=true \ + --enable_graph=true \ + --acl_graph_decode_batch_size_limit=4 \ + --draft_model=$DRAFT_MODEL_PATH \ + --draft_devices="npu:$DEVICE" \ + --num_speculative_tokens=3 \ + --ep_size=32 \ + --dp_size=4 \ + --rank_tablefile=/yourPath/ranktable.json \ + --tool_call_parser=auto \ + > $LOG_FILE 2>&1 & +done +``` + +#### Node1 (worker) + +```bash +MASTER_NODE_ADDR="11.87.49.110:19990" +LOCAL_HOST="11.87.49.111" +START_PORT=15890 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=32 +LOCAL_NODES=16 +export HCCL_IF_BASE_PORT=48439 +unset HCCL_OP_EXPANSION_MODE + +for (( i=0; i<$LOCAL_NODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$((i + LOCAL_NODES)) \ + --max_memory_utilization=0.85 \ + --max_tokens_per_batch=8192 \ + --max_seqs_per_batch=128 \ + --block_size=128 \ + --enable_prefix_cache=true \ + --enable_chunked_prefill=true \ + --communication_backend="hccl" \ + --enable_schedule_overlap=true \ + --enable_graph=true \ + --acl_graph_decode_batch_size_limit=4 \ + --draft_model=$DRAFT_MODEL_PATH \ + --draft_devices="npu:$DEVICE" \ + --num_speculative_tokens=3 \ + --ep_size=32 \ + --dp_size=4 \ + --rank_tablefile=/yourPath/ranktable.json \ + --tool_call_parser=auto \ + > $LOG_FILE 2>&1 & +done +``` + +### Rank Table Examples + +[A3 rank table configuration](https://www.hiascend.com/document/detail/zh/canncommercial/900/API/hcclug/hcclug_000066.html) + +[A2 rank table configuration](https://www.hiascend.com/document/detail/zh/canncommercial/900/API/hcclug/hcclug_000067.html) + +Note that the rank table formats differ between A3 and A2. + +## View Device NUMA Affinity + +Command: + +```bash +npu-smi info -t topo +``` + +In the preceding commands: + +```bash +numactl -C $((DEVICE*12))-$((DEVICE*12+11)) +``` + +indicates that the process is bound to the corresponding affinity cores. You can modify the bound core IDs according to the machine. + +## EX3. GLM-5 Weight Quantization + +GLM-5.2 quantization guidance will be updated later. + +### Install msmodelslim + +```bash +pip install transformers==5.2.0 + +git clone https://gitcode.com/Ascend/msmodelslim.git +cd msmodelslim +bash install.sh +``` + +### Run Quantization + +```bash +msmodelslim quant \ + --model_path ${MODEL_PATH} \ + --save_path ${SAVE_PATH} \ + --device npu:0 \ + --model_type GLM-5 \ + --quant_type w8a8 \ + --trust_remote_code True +``` + +## PD Disaggregation + +### Install etcd and xllm-service + +#### PD Disaggregated Deployment + +`xllm` supports PD disaggregated deployment. This must be used together with another open-source library, [xllm service](https://github.com/xLLM-AI/xllm-service). + +##### xLLM Service Dependencies + +First, download and install `xllm service`, similar to installing and building `xllm`: + +```bash +git clone https://github.com/xLLM-AI/xllm-service.git +cd xllm-service +git submodule init +git submodule update +``` + +##### Install etcd + +`xllm_service` depends on [etcd](https://github.com/etcd-io/etcd). Use the official etcd [installation script](https://github.com/etcd-io/etcd/releases) to install it. The default installation path used by the script is `/tmp/etcd-download-test/etcd`. You can manually modify the installation path in the script, or move it manually after the script finishes: + +```bash +mv /tmp/etcd-download-test/etcd /path/to/your/etcd +``` + +##### Build xLLM Service + +Apply the patch first: + +```bash +sh prepare.sh +``` + +Then build: + +```bash +mkdir -p build +cd build +cmake .. +make -j 8 +cd .. +``` + +:::caution[Possible Errors] +You may encounter installation errors related to `boost-locale` and `boost-interprocess`: `vcpkg-src/packages/boost-locale_x64-linux/include: No such file or directory`, `/vcpkg-src/packages/boost-interprocess_x64-linux/include: No such file or directory`. +Reinstall these packages with `vcpkg`: +```bash +/path/to/vcpkg remove boost-locale boost-interprocess +/path/to/vcpkg install boost-locale:x64-linux +/path/to/vcpkg install boost-interprocess:x64-linux +``` + +::: + +### Run PD Disaggregation + +Start etcd: + +```bash +./etcd-download-test/etcd --listen-peer-urls 'http://localhost:2390' --listen-client-urls 'http://localhost:2389' --advertise-client-urls 'http://localhost:2391' +``` + +For cross-machine configuration, refer to the following etcd command: + +```bash +/tmp/etcd-download-test/etcd --listen-peer-urls 'http://0.0.0.0:3390' --listen-client-urls 'http://0.0.0.0:3389' --advertise-client-urls 'http://11.87.191.82:3389' +``` + +Start xllm service: + +```bash +ENABLE_DECODE_RESPONSE_TO_SERVICE=true ./xllm_master_serving --etcd_addr="127.0.0.1:12389" --http_server_port 28888 --rpc_server_port 28889 --tokenizer_path=/export/home/models/GLM-5-W8A8/ +``` + +For cross-machine configuration, start xllm service with: + +```bash +ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_master_serving --etcd_addr="11.87.191.82:3389" --http_server_port 38888 --rpc_server_port 38889 --tokenizer_path=/export/home/models/GLM-5-W8A8/ +``` + +- Start the Prefill instance +```bash + BATCH_SIZE=256 + # Maximum inference batch size + XLLM_PATH="./myxllm/xllm/build/xllm/core/server/xllm" + # Inference entry binary path, which is the build artifact from the previous step + MODEL_PATH=/export/home/models/GLM-5-w8a8/ + # Model path, here using the int-quantized GLM-5 + DRAFT_MODEL_PATH=/export/home/models/GLM-5-MTP/ + + MASTER_NODE_ADDR="11.87.49.110:10015" + LOCAL_HOST="11.87.49.110" + # Service port + START_PORT=18994 + START_DEVICE=0 + LOG_DIR="logs" + NNODES=16 + + for (( i=0; i<$NNODES; i++ )) + do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + --model $MODEL_PATH --model_id glmmoe \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + --max_memory_utilization=0.86 \ + --max_tokens_per_batch=5000 \ + --max_seqs_per_batch=$BATCH_SIZE \ + --communication_backend=hccl \ + --enable_schedule_overlap=true \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_graph=true \ + --draft_model $DRAFT_MODEL_PATH \ + --draft_devices="npu:$DEVICE" \ + --num_speculative_tokens 1 \ + --tool_call_parser=auto \ + --enable_disagg_pd=true \ + --instance_role=PREFILL \ + --etcd_addr=$LOCAL_HOST:3389 \ + --transfer_listen_port=$((36100 + i)) \ + --disagg_pd_port=8877 \ + > $LOG_FILE 2>&1 & + done + + # --etcd_addr=$LOCAL_HOST:3389 Refer to the advertise-client-urls configuration in etcd. + # --instance_role=DECODE PD configuration: DECODE or PREFILL. + ``` + +- Start the Decode instance + + ```bash + BATCH_SIZE=256 + # Maximum inference batch size + XLLM_PATH="./myxllm/xllm/build/xllm/core/server/xllm" + # Inference entry binary path, which is the build artifact from the previous step + MODEL_PATH=/export/home/models/GLM-5-w8a8/ + # Model path, here using the int-quantized GLM-5 + DRAFT_MODEL_PATH=/export/home/models/GLM-5-MTP/ + + MASTER_NODE_ADDR="11.87.49.110:10015" + LOCAL_HOST="11.87.49.110" + # Service port + START_PORT=18994 + START_DEVICE=0 + LOG_DIR="logs" + NNODES=16 + + for (( i=0; i<$NNODES; i++ )) + do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + --model $MODEL_PATH --model_id glmmoe \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + --max_memory_utilization=0.86 \ + --max_tokens_per_batch=5000 \ + --max_seqs_per_batch=$BATCH_SIZE \ + --communication_backend=hccl \ + --enable_schedule_overlap=true \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_graph=true \ + --draft_model $DRAFT_MODEL_PATH \ + --draft_devices="npu:$DEVICE" \ + --num_speculative_tokens 1 \ + --tool_call_parser=auto \ + --enable_disagg_pd=true \ + --instance_role=DECODE \ + --etcd_addr=$LOCAL_HOST:3389 \ + --transfer_listen_port=$((36100 + i)) \ + --disagg_pd_port=8877 \ + > $LOG_FILE 2>&1 & + done + + # --etcd_addr=$LOCAL_HOST:3389 Refer to the advertise-client-urls configuration in etcd. + # --instance_role=DECODE PD configuration: DECODE or PREFILL. + ``` + + Notes: + +- PD disaggregation needs to read `/etc/hccn.conf`. Make sure this file on the physical machine is mounted into the container. + +- `etcd_addr` must be the same as the `etcd_addr` used by `xllm_service`. + The test command is similar to the one above. Note that for `curl http://localhost:{PORT}/v1/chat/completions ...`, `PORT` should be the `http_server_port` used to start xLLM service. + +- When deploying P or Q across multiple machines, such as deploying two P instances, add `--rank_tablefile` to complete communication. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5_1.md b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5_1.md new file mode 100644 index 0000000000..999ce15dbb --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5_1.md @@ -0,0 +1,6 @@ +--- +title: "GLM-5.1" +description: "GLM-5.1 inference recipes are consolidated into the GLM-5 series guide" +--- + +GLM-5.1 inference recipes are consolidated into [GLM-5 / GLM-5.1 / GLM-5.2](/en/cookbook/autoregressive_models/glm/glm_5/). diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5_2.md b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5_2.md new file mode 100644 index 0000000000..8461d966b3 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5_2.md @@ -0,0 +1,6 @@ +--- +title: "GLM-5.2" +description: "GLM-5.2 inference recipes are consolidated into the GLM-5 series guide" +--- + +GLM-5.2 inference recipes are consolidated into [GLM-5 / GLM-5.1 / GLM-5.2](/en/cookbook/autoregressive_models/glm/glm_5/). diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2.md b/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2.md new file mode 100644 index 0000000000..fe4559cdba --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2.md @@ -0,0 +1,8 @@ +--- +title: "Kimi2" +description: "CookBook recipe placeholder for Kimi2 autoregressive model inference" +--- + +This section will collect Kimi2 autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md b/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md new file mode 100644 index 0000000000..00c2156c32 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md @@ -0,0 +1,300 @@ +--- +title: "Kimi-K2.5 / Kimi-K2.6" +sidebar: + order: 2 +--- + +- Source code: [https://github.com/xLLM-AI/xllm](https://github.com/xLLM-AI/xllm) +- Available in China: [https://gitcode.com/xLLM-AI/xllm](https://gitcode.com/xLLM-AI/xllm) +- Kimi-K2.5 W8A8 weight download: [modelscope-Kimi-K2.5-W8A8-xLLM](https://www.modelscope.cn/models/Eco-Tech/Kimi-K2.5-W8A8-xLLM) +- Kimi-K2.6 W8A8 weight download: [modelscope-Kimi-K2.6-w8a8-xllm](https://www.modelscope.cn/models/Eco-Tech/Kimi-K2.6-w8a8-xllm) + +P.S. Kimi-K2.5 and Kimi-K2.6 use the same model architecture. The following sections use Kimi-K2.5 as an example to describe the overall deployment process. + +## 0. Weight Preparation + +### Download Weights from ModelScope + +```bash +export MODELSCOPE_CACHE=path-to-model # Default: ~/.cache/modelscope/hub +pip install modelscope +modelscope download --model Eco-Tech/Kimi-K2.5-W8A8-xLLM +``` + +## 1. Pull the Image Environment + +First, download the image provided by xLLM: + +```bash +# A3 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-20260429 +``` + +Then create the corresponding container: + +```bash +sudo docker run -it --ipc=host -u 0 --privileged --name xllm_kimi_k25 --network=host \ + -v /var/queue_schedule:/var/queue_schedule \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ + -v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \ + -v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \ + -v /var/log/npu/slog/:/var/log/npu/slog \ + -v ~/.ssh:/root/.ssh \ + -v /var/log/npu/profiling/:/var/log/npu/profiling \ + -v /var/log/npu/dump/:/var/log/npu/dump \ + -v /runtime/:/runtime/ -v /etc/hccn.conf:/etc/hccn.conf \ + -v /export/home:/export/home \ + -v /home/:/home/ \ + -w /export/home \ + quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-20260429 +``` + +## 2. Pull the Source Code and Build + +Download the official repository and module dependencies: + +```bash +git clone https://github.com/xLLM-AI/xllm.git +cd xllm +git checkout main +git submodule init +git submodule update +``` + +Download and install dependencies: + +```bash +pip install --upgrade pre-commit +yum install numactl +``` + +Run the build to generate the executable under `build/`: + +```bash +python setup.py build +``` + +Build artifact path: `build/xllm/core/server/xllm` + +## 3. Start the Model + +### If the service is being started for the first time after the machine has rebooted, run the following script first to initialize the devices + +If this is skipped and the NPU has not been initialized, the xLLM process may fail to start. + +```bash +python -c "import torch_npu +for i in range(16):torch_npu.npu.set_device(i)" +``` + +### Environment Variables + +```bash +##### 1. Configure dependency path environment variables +export PYTHON_INCLUDE_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])')" +export PYTHON_LIB_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])')" +export PYTORCH_NPU_INSTALL_PATH=/usr/local/libtorch_npu/ +export PYTORCH_INSTALL_PATH="$(python3 -c 'import torch, os; print(os.path.dirname(os.path.abspath(torch.__file__)))')" +export LIBTORCH_ROOT="$(python3 -c 'import torch, os; print(os.path.dirname(os.path.abspath(torch.__file__)))')" + +export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/xllm/op_api/lib/:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/usr/local/libtorch_npu/lib:$LD_LIBRARY_PATH +export LD_PRELOAD=/usr/lib64/libjemalloc.so.2:$LD_PRELOAD + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +##### 2. Configure log-related environment variables +rm -rf /root/atb/log/ +rm -rf /root/ascend/log/ +rm -rf core.* +export ASDOPS_LOG_LEVEL=ERROR +export ASDOPS_LOG_TO_STDOUT=1 +export ASDOPS_LOG_TO_FILE=1 + +##### 3. Configure performance and communication-related environment variables +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export NPU_MEMORY_FRACTION=0.96 +export ATB_WORKSPACE_MEM_ALLOC_ALG_TYPE=3 +export ATB_WORKSPACE_MEM_ALLOC_GLOBAL=1 + +export OMP_NUM_THREADS=12 +export ALLOW_INTERNAL_FORMAT=1 + +export ATB_LAYER_INTERNAL_TENSOR_REUSE=1 +export ATB_LLM_ENABLE_AUTO_TRANSPOSE=0 +export ATB_CONVERT_NCHW_TO_ND=1 +export ATB_LAUNCH_KERNEL_WITH_TILING=1 +export ATB_OPERATION_EXECUTE_ASYNC=2 +export ATB_CONTEXT_WORKSPACE_SIZE=0 +export INF_NAN_MODE_ENABLE=1 +export HCCL_EXEC_TIMEOUT=0 +export HCCL_CONNECT_TIMEOUT=7200 +export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_IF_BASE_PORT=2864 +``` + +## Startup Command - Kimi_k25 (two machines, 16 cards, 32 dies, tp=4, dp=8, ep=32) + +### Node0 (master) + +```bash +MASTER_NODE_ADDR="11.87.49.110:19990" +LOCAL_HOST="11.87.49.110" +START_PORT=15890 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=32 +LOCAL_NODES=16 +export HCCL_IF_BASE_PORT=48439 +unset HCCL_OP_EXPANSION_MODE + +for (( i=0; i<$LOCAL_NODES; i++ ))do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + --max_memory_utilization=0.85 \ + --max_tokens_per_batch=8192 \ + --max_seqs_per_batch=20 \ + --block_size=128 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --communication_backend="hccl" \ + --enable_schedule_overlap=true \ + --enable_graph=false \ + --enable_shm=true \ + --ep_size=32 \ + --dp_size=8 \ + --input_shm_size=4096 \ + --rank_tablefile=/yourPath/ranktable.json \ + > $LOG_FILE 2>&1 & +done +``` + +#### Node1 (worker) + +```bash +MASTER_NODE_ADDR="11.87.49.110:19990" +LOCAL_HOST="11.87.49.111" +START_PORT=15890 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=32 +LOCAL_NODES=16 +export HCCL_IF_BASE_PORT=48439 +unset HCCL_OP_EXPANSION_MODE + +for (( i=0; i<$LOCAL_NODES; i++ ))do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$((i + LOCAL_NODES)) \ + --max_memory_utilization=0.85 \ + --max_tokens_per_batch=8192 \ + --max_seqs_per_batch=20 \ + --block_size=128 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --communication_backend="hccl" \ + --enable_schedule_overlap=true \ + --enable_graph=false \ + --enable_shm=true \ + --ep_size=32 \ + --dp_size=8 \ + --input_shm_size=4096 \ + --rank_tablefile=/yourPath/ranktable.json \ +done +``` + +#### ranktable Example + +ranktable configuration guide: [https://www.hiascend.com/document/detail/zh/canncommercial/83RC1/hccl/hcclug/hcclug_000014.html](https://www.hiascend.com/document/detail/zh/canncommercial/83RC1/hccl/hcclug/hcclug_000014.html) + +```bash +ln -s /usr/local/Ascend/driver/tools/hccn_tool /usr/sbin/ + +#device_ip +for i in {0..15};do hccn_tool -i $i -vnic -g; done + +#super_device_id +for i in {0..7};do for j in {0..1}; do npu-smi info -t spod-info -i $i -c $j; done; done +``` + +```json +{ + "status": "completed", + "version": "1.2", + "server_count": "2", + "server_list": [ + { + "server_id": "10.87.191.98", + "host_nic_ip": "reserve", + "host_ip": "10.87.191.98", + "container_ip": "10.87.191.98", + "device": [ + { + "device_id": "0", + "device_ip": "192.24.2.199", + "super_device_id": "100663296", + "rank_id": "16" + }, + ... + { + "device_id": "15", + "device_ip": "192.24.3.184", + "super_device_id": "102563855", + "rank_id": "31" + } + ] + }, + { + "server_id": "10.87.191.102", + "host_nic_ip": "reserve", + "host_ip": "10.87.191.102", + "container_ip": "10.87.191.102", + "device": [ + { + "device_id": "0", + "device_ip": "192.28.2.199", + "super_device_id": "117440512", + "rank_id": "0" + }, + ... + { + "device_id": "15", + "device_ip": "192.28.3.184", + "super_device_id": "119341071", + "rank_id": "15" + } + ] + } + ], + "super_pod_list": [ + { + "super_pod_id": "2", + "server_list": [ + { + "server_id": "10.87.191.98" + }, + { + "server_id": "10.87.191.102" + } + ] + } + ] +} +``` + +When the log contains `"Application startup complete."`, the service has started successfully. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md b/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md new file mode 100644 index 0000000000..89bb876f5d --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md @@ -0,0 +1,198 @@ +--- +title: "MiniMax-M2.7" +sidebar: + order: 3 +--- + ++ Source code: https://github.com/xLLM-AI/xllm + ++ Available in China: https://gitcode.com/xLLM-AI/xllm + ++ Weight download: [modelscope-MiniMax-M2.7](https://www.modelscope.cn/models/MiniMax/MiniMax-M2.7) ++ Offline dequantized weights: [modelscope-Minimax2.7-BF16-xLLM](https://modelscope.cn/models/Eco-Tech/Minimax2.7-BF16-xLLM) + +## 0. Weight Preparation + +The original MiniMax-M2.7 weights are in FP8 format. xLLM supports the following three loading methods: + +### Method 1: Load FP8 weights directly (online dequantization) + +Use the original FP8 weight path directly. xLLM will dequantize FP8 to BF16 during inference, so no additional preprocessing is required. + +```bash +MODEL_PATH=/path/to/MiniMax-M2.7/ +``` + +### Method 2: Offline dequantization + +Use the tool script to convert FP8 weights to BF16 in advance to avoid the extra overhead of online dequantization: + +```bash +python tools/dequant_minimax_fp8.py --input-dir /path/to/MiniMax-M2.7/ --output-dir /path/to/MiniMax-M2.7-bf16/ +``` + +### Method 3: Download pre-converted BF16 weights + +Download the dequantized BF16 weights directly: + +```bash +git clone https://www.modelscope.cn/Eco-Tech/Minimax2.7-BF16-xLLM.git +``` + +## 1. Pull the Image Environment + +First, download the image provided by xLLM: + +```bash +# A3 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-20260429 +``` + +Then create the corresponding container: + +```bash +sudo docker run -it --ipc=host -u 0 --privileged --name xllm_minimax --network=host \ + -v /var/queue_schedule:/var/queue_schedule \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ + -v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \ + -v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \ + -v /var/log/npu/slog/:/var/log/npu/slog \ + -v ~/.ssh:/root/.ssh \ + -v /var/log/npu/profiling/:/var/log/npu/profiling \ + -v /var/log/npu/dump/:/var/log/npu/dump \ + -v /runtime/:/runtime/ -v /etc/hccn.conf:/etc/hccn.conf \ + -v /export/home:/export/home \ + -v /home/:/home/ \ + -w /export/home \ + quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-20260429 +``` + +## 2. Pull the Source Code and Build + +Download the official repository and module dependencies: + +```bash +git clone https://github.com/xLLM-AI/xllm.git +cd xllm +git checkout preview/minimax-minimal +git submodule init +git submodule update +``` + +Download and install dependencies: + +```bash +pip install --upgrade pre-commit +yum install numactl +``` + +Run the build to generate the executable under `build/`: + +```bash +python setup.py build +``` + +Build artifact path: `build/xllm/core/server/xllm` + +## 3. Start the Model + +### If the service is being started for the first time after the machine has rebooted, initialize the devices first + +If this is skipped and the NPU has not been initialized, the xLLM process may fail to start. + +```bash +python -c "import torch_npu +for i in range(16):torch_npu.npu.set_device(i)" +``` + +### Environment Variables + +```bash +##### 1. Configure dependency path environment variables +export PYTHON_INCLUDE_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])')" +export PYTHON_LIB_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])')" +export PYTORCH_NPU_INSTALL_PATH=/usr/local/libtorch_npu/ +export PYTORCH_INSTALL_PATH="$(python3 -c 'import torch, os; print(os.path.dirname(os.path.abspath(torch.__file__)))')" +export LIBTORCH_ROOT="$(python3 -c 'import torch, os; print(os.path.dirname(os.path.abspath(torch.__file__)))')" + +export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/xllm/op_api/lib/:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/usr/local/libtorch_npu/lib:$LD_LIBRARY_PATH +export LD_PRELOAD=/usr/lib64/libjemalloc.so.2:$LD_PRELOAD + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +##### 2. Configure log-related environment variables +rm -rf /root/atb/log/ +rm -rf /root/ascend/log/ +rm -rf core.* +export ASDOPS_LOG_LEVEL=ERROR +export ASDOPS_LOG_TO_STDOUT=1 +export ASDOPS_LOG_TO_FILE=1 + +##### 3. Configure performance and communication-related environment variables +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export NPU_MEMORY_FRACTION=0.96 +export ATB_WORKSPACE_MEM_ALLOC_ALG_TYPE=3 +export ATB_WORKSPACE_MEM_ALLOC_GLOBAL=1 + +export OMP_NUM_THREADS=12 +export ALLOW_INTERNAL_FORMAT=1 + +export ATB_LAYER_INTERNAL_TENSOR_REUSE=1 +export ATB_LLM_ENABLE_AUTO_TRANSPOSE=0 +export ATB_CONVERT_NCHW_TO_ND=1 +export ATB_LAUNCH_KERNEL_WITH_TILING=1 +export ATB_OPERATION_EXECUTE_ASYNC=2 +export ATB_CONTEXT_WORKSPACE_SIZE=0 +export INF_NAN_MODE_ENABLE=1 +export HCCL_EXEC_TIMEOUT=0 +export HCCL_CONNECT_TIMEOUT=7200 +export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_IF_BASE_PORT=2864 +``` + +## Startup Command - MiniMax-M2.7 (single machine, 16 cards, TP=16) + +```bash +BATCH_SIZE=256 +# Maximum inference batch size +XLLM_PATH="build/xllm/core/server/xllm" +# Inference entry binary path, which is the build artifact from the previous step +MODEL_PATH=/path/to/MiniMax-M2.7/ +# Model path + +MASTER_NODE_ADDR="10.143.3.204:10015" +LOCAL_HOST="10.143.3.204" +# Service port +START_PORT=18994 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=16 + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + --max_memory_utilization=0.90 \ + --max_tokens_per_batch=8192 \ + --max_seqs_per_batch=$BATCH_SIZE \ + --communication_backend=hccl \ + --enable_chunked_prefill=false \ + --enable_prefix_cache=false \ + --enable_schedule_overlap=false \ + --enable_graph=false \ + --enable_atb_spec_kernel=false \ + > $LOG_FILE 2>&1 & +done +``` diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen2_5_vl.md b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen2_5_vl.md new file mode 100644 index 0000000000..d06257dc4c --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen2_5_vl.md @@ -0,0 +1,8 @@ +--- +title: "Qwen2.5-VL" +description: "CookBook recipe placeholder for Qwen2.5-VL autoregressive model inference" +--- + +This section will collect Qwen2.5-VL autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3.md b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3.md new file mode 100644 index 0000000000..8fb86750f0 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3.md @@ -0,0 +1,8 @@ +--- +title: "Qwen3" +description: "CookBook recipe placeholder for Qwen3 autoregressive model inference" +--- + +This section will collect Qwen3 autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md new file mode 100644 index 0000000000..0166b86be0 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md @@ -0,0 +1,158 @@ +--- +title: "Qwen3.5" +sidebar: + order: 1 +--- + ++ Source code: https://github.com/xLLM-AI/xllm + ++ Available in China: https://gitcode.com/xLLM-AI/xllm + ++ Weight download: [modelscope-Qwen3.5-27B](https://www.modelscope.cn/models/Qwen/Qwen3.5-27B) + +## 1. Pull the Image Environment + +First, download the image provided by xLLM: + +```bash +# A3 arm (CANN 9) +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-cann9-20260605 +``` + +Then create the corresponding container: + +```bash +docker run -it -d \ + --ipc=host \ + -u 0 \ + --privileged \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --name xllm_qwen35 \ + --network=host \ + --device=/dev/davinci0 \ + --device=/dev/davinci_manager \ + --device=/dev/devmm_svm \ + --device=/dev/hisi_hdc \ + -v /var/queue_schedule:/var/queue_schedule \ + -v /mnt/cfs/9n-das-admin/llm_models:/mnt/cfs/9n-das-admin/llm_models \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ + -v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \ + -v /usr/local/sbin/:/usr/local/sbin/ \ + -v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \ + -v /var/log/npu/slog/:/var/log/npu/slog \ + -v /var/log/npu/profiling/:/var/log/npu/profiling \ + -v /var/log/npu/dump/:/var/log/npu/dump \ + -v /export/home:/export/home \ + -v ~/.ssh:/root/.ssh \ + -v /home/:/home/ \ + -v /runtime/:/runtime/ \ + -w /home \ + quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-cann9-20260605 +``` + +## 2. Pull the Source Code and Build + +Download the official repository and module dependencies: + +```bash +git clone https://github.com/xLLM-AI/xllm.git +cd xllm +pip install pre-commit +pre-commit install +git submodule update --init --recursive +``` + +Run the build to generate the executable under `build/`: + +```bash +python setup.py build +``` + +Build artifact path: `build/xllm/core/server/xllm` + +## 3. Start the Model + +### Environment Variables + +```bash +# 1. Configure dependency path environment variables +export ASDOPS_LOG_TO_STDOUT=0 +export ASDOPS_LOG_LEVEL=3 +export PYTHON_INCLUDE_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])' | tail -n 1)" +export PYTHON_LIB_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])' | tail -n 1)" +export PYTORCH_NPU_INSTALL_PATH=/usr/local/libtorch_npu/ +export PYTORCH_INSTALL_PATH="$(python3 -c 'import site, os; print(os.path.join(site.getsitepackages()[0], "torch"))')" +export LIBTORCH_ROOT="$PYTORCH_INSTALL_PATH" +export LD_LIBRARY_PATH=/usr/local/libtorch_npu/lib:$LD_LIBRARY_PATH + +# 2. Load environment +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export ASCEND_RT_VISIBLE_DEVICES=14,15 +export ASDOPS_LOG_TO_STDOUT=1 +export ASDOPS_LOG_LEVEL=0 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export NPU_MEMORY_FRACTION=0.90 +export ATB_WORKSPACE_MEM_ALLOC_ALG_TYPE=3 +export ATB_WORKSPACE_MEM_ALLOC_GLOBAL=1 +export OMP_NUM_THREADS=12 +export HCCL_CONNECT_TIMEOUT=7200 +export INF_NAN_MODE_ENABLE=0 +export INF_NAN_MODE_FORCE_DISABLE=1 + +# 3. Clean up old logs +LOG_DIR="log" +mkdir -p $LOG_DIR +``` + +:::note +Qwen3.5 does not currently support TP=16. +::: + +## Startup Command - Qwen3.5-27B (2 cards, TP=2, speculative decoding) + +```bash +MODEL_PATH="/path/to/Qwen3.5-27B" +DRAFT_MODEL_PATH="/path/to/Qwen3.5-27B-mtp" + +MASTER_NODE_ADDR=":32764" +START_PORT=18076 +START_DEVICE=0 +NNODES=2 + +export HCCL_IF_BASE_PORT=53433 + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + ./xllm/build/xllm/core/server/xllm \ + --model $MODEL_PATH \ + --devices="npu:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --max_memory_utilization=0.7 \ + --max_tokens_per_batch=32768 \ + --max_seqs_per_batch=8 \ + --block_size=128 \ + --communication_backend="lccl" \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=true \ + --enable_schedule_overlap=true \ + --enable_graph=true \ + --node_rank=$i \ + --enable_shm=true \ + --task="generate" \ + --max_concurrent_requests=8 \ + --backend llm \ + --draft_model $DRAFT_MODEL_PATH \ + --draft_devices="npu:$DEVICE" \ + --num_speculative_tokens 3 \ + >> $LOG_FILE 2>&1 & +done +``` diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_next.md b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_next.md new file mode 100644 index 0000000000..788376c84a --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_next.md @@ -0,0 +1,8 @@ +--- +title: "Qwen3-Next" +description: "CookBook recipe placeholder for Qwen3-Next autoregressive model inference" +--- + +This section will collect Qwen3-Next autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_vl.md b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_vl.md new file mode 100644 index 0000000000..8ac3296306 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_vl.md @@ -0,0 +1,8 @@ +--- +title: "Qwen3-VL" +description: "CookBook recipe placeholder for Qwen3-VL autoregressive model inference" +--- + +This section will collect Qwen3-VL autoregressive model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/diffusion_models.md b/docs/src/content/docs/en/cookbook/diffusion_models.md new file mode 100644 index 0000000000..f7b948cb4e --- /dev/null +++ b/docs/src/content/docs/en/cookbook/diffusion_models.md @@ -0,0 +1,14 @@ +--- +title: "Diffusion Models" +description: "CookBook recipes for diffusion model inference with xLLM" +--- + +This section collects CookBook recipes for serving diffusion models with xLLM. + +Content will be added as the recipes are organized. + +## Model Families + +- [Flux](/en/cookbook/diffusion_models/flux/flux/) +- [Wan](/en/cookbook/diffusion_models/wan/wan2_1/) +- [Qwen-Image](/en/cookbook/diffusion_models/qwen_image/qwen_image/) diff --git a/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux.md b/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux.md new file mode 100644 index 0000000000..7fbdfa9e9a --- /dev/null +++ b/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux.md @@ -0,0 +1,8 @@ +--- +title: "Flux" +description: "CookBook recipe placeholder for Flux diffusion model inference" +--- + +This section will collect Flux diffusion model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md b/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md new file mode 100644 index 0000000000..2d98560a70 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md @@ -0,0 +1,517 @@ +--- +title: "Flux2" +description: "CookBook recipe placeholder for Flux2 diffusion model inference" +sidebar: + order: 3 +--- + +This section will collect Flux2 diffusion model inference recipes for xLLM. + ++ Source code: https://github.com/xLLM-AI/xllm + ++ Available in China: https://gitcode.com/xLLM-AI/xllm + ++ Weight download: [modelscope-FLUX.2-dev](https://www.modelscope.cn/models/black-forest-labs/FLUX.2-dev/) + +## 1. Pull the Image Environment + +First, download the image provided by xLLM: + +```bash +# A2 x86 +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-x86-cann9-20260605 +# A2 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-arm-cann9-20260605 +# A3 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-cann9-20260605 +``` + +Then create the corresponding container: + +```bash +IMAGE=quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-cann9-20260605 +CONTAINER=mydocker + +docker run \ +--name $CONTAINER \ +--privileged \ +--network=host \ +--ipc=host \ +--device /dev/davinci_manager \ +--device /dev/devmm_svm \ +--device /dev/hisi_hdc \ +-v /usr/local/dcmi:/usr/local/dcmi \ +-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \ +-v /usr/local/Ascend/driver/:/usr/local/Ascend/driver/ \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /root/.cache:/root/.cache \ +-v /export/home:/export/home \ +-v /home/:/home/ \ +-w /export/home \ +-itd $IMAGE bash +docker exec -it $CONTAINER bash +``` + +## 2. Pull the Source Code and Build + +Download the official repository and module dependencies: + +```bash +git clone https://github.com/xLLM-AI/xllm.git +cd xllm + +``` + +Download and install dependencies: + +```bash +pip install pre-commit +pre-commit install + +git submodule update --init --recursive +``` + +Run the build to generate the executable `build/xllm/core/server/xllm` under `build/`: + +```bash +python setup.py build +``` + +## 3. Start the Model + +### If the service is being started for the first time after the machine has rebooted, initialize the devices first + +If this is skipped and the NPU has not been initialized, the xLLM process may fail to start. + +```bash +python -c "import torch_npu +for i in range(16):torch_npu.npu.set_device(i)" +``` + +Flux2 serving in xLLM uses a two-stage mode: you need to start the text-encoder component and the DiT component separately, then trigger the full Flux2 inference pipeline via the Python embedding script. Additionally, Flux2 supports TP, SP, and dit_cache features (TaylorSeer, ResidualCache), but does not currently support the chunked prefill feature. + +### 1. Start the text-encoder component +#### Environment Variables + +```bash +##### 1. Configure dependency path environment variables +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +##### 2. Configure log-related environment variables +rm -rf core.* +export ASCEND_MODULE_LOG_LEVEL=ATB=0 +export ASDOPS_LOG_TO_FILE=1 +export ASCEND_SLOG_PRINT_TO_STDOUT=1 + +##### 3. Configure performance and communication-related environment variables +export HCCL_IF_BASE_PORT=43432 # HCCL communication base port +``` + +#### Startup Command - Start the Flux2 text-encoder component (single machine, 1 card with 2 dies, TP=2) + +```bash +MODEL_PATH="/path/to/flux2/text_encoder/" # text_encoder path +MASTER_NODE_ADDR="127.0.0.1:9748" # Master node address (must be consistent globally) +START_PORT=18001 # Service starting port +START_DEVICE=2 # Starting logical device ID +LOG_DIR="log" # Log directory +NNODES=2 # Number of nodes (this script starts 1 process) + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/mistral_node_$i.log" + ./build/xllm/core/server/xllm \ + --model $MODEL_PATH \ + --devices="npu:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --max_memory_utilization=0.86 \ + --block_size=128 \ + --tp_size=2 \ + --communication_backend="hccl" \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_schedule_overlap=true \ + --enable_return_mm_full_embeddings=true \ + --enable_mistral_prompt_to_message=true \ + --task="embed" \ + --enable_shm=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` +### 2. Start the DiT component +#### Environment Variables + +```bash +##### 1. Configure dependency path environment variables +export PYTHON_INCLUDE_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])')" +export PYTHON_LIB_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])')" +export PYTORCH_NPU_INSTALL_PATH=/usr/local/libtorch_npu/ # NPU PyTorch path +export PYTORCH_INSTALL_PATH="$(python3 -c 'import torch, os; print(os.path.dirname(os.path.abspath(torch.__file__)))')" # PyTorch installation path +export LIBTORCH_ROOT="$PYTORCH_INSTALL_PATH" # LibTorch path +export LD_LIBRARY_PATH=/usr/local/libtorch_npu/lib:$LD_LIBRARY_PATH # Add NPU library path + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +##### 2. Configure log-related environment variables +\rm -rf core.* +\rm -rf log/dit_node_*.log + +export ASDOPS_LOG_LEVEL=ERROR +export ASDOPS_LOG_TO_STDOUT=1 +export ASDOPS_LOG_TO_FILE=1 + +##### 3. Configure performance and communication-related environment variables +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export NPU_MEMORY_FRACTION=0.98 +export ATB_WORKSPACE_MEM_ALLOC_ALG_TYPE=3 +export ATB_WORKSPACE_MEM_ALLOC_GLOBAL=1 +export OMP_NUM_THREADS=12 +export HCCL_CONNECT_TIMEOUT=7200 +export INF_NAN_MODE_ENABLE=0 +export INF_NAN_MODE_FORCE_DISABLE=1 +export HCCL_IF_BASE_PORT=43432 # HCCL communication base port +``` + +#### Startup Command - Flux2 DiT component (single machine, 1 card with 2 dies, TP=2) + +```bash +MASTER_NODE_ADDR="127.0.0.1:8999" # Master node address (must be consistent globally) +START_PORT=18018 # Service starting port +START_DEVICE=4 # Starting NPU logical device ID +LOG_DIR="log" # Log directory +NNODES=2 # Number of nodes (this script starts 2 processes) + +for (( i=0; i<2; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/dit_node_$i.log" + ./build/xllm/core/server/xllm \ + --model="/path/to/flux2/" \ + --max_memory_utilization=0.6 \ + --backend="dit" \ + --tp_size=2 \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --port $PORT \ + --communication_backend="hccl" \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_schedule_overlap=false \ + --use_contiguous_input_buffer=false \ + --dit_debug_print=true \ + --enable-shm=true \ + --node_rank=$i > $LOG_FILE 2>&1 & +done +``` +### 3. Run the Python embedding script + +```bash +# -*- coding: utf-8 -*- +import sys +import json +from typing import Callable, Optional, Union +from safetensors.torch import load_file +import torch +import os +import base64 +import requests +import argparse +import PIL.Image +import PIL.ImageOps +import torch +import math +import io +import numpy as np +import time + +from transformers import AutoProcessor, AutoTokenizer +CONDITION_IMAGE_SIZE = 384 * 384 +VAE_IMAGE_SIZE = 1024 * 1024 + +def load_tensor( + image: Union[str, PIL.Image.Image], + convert_method: Optional[Callable[[PIL.Image.Image], PIL.Image.Image]] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + """ + Load `image` (URL / local path / PIL.Image) and convert to torch.Tensor. + + Args: + image (str or PIL.Image.Image): URL (http/https) or filesystem path, or PIL Image. + convert_method (Callable, optional): If provided, apply a custom conversion to the PIL.Image after reading. + If None, `.convert("RGB")` is called by default. + device (torch.device, optional): The device to place the tensor on (e.g. torch.device('cuda')). + If None, no device transfer is performed (defaults to CPU). + dtype (torch.dtype, optional): The dtype of the returned tensor (e.g. torch.float32). If None, torch.float32 is used. + + Returns: + torch.Tensor: shape [C, H, W], dtype float, values in [0,1], on the specified device (if provided). + """ + # Read as PIL.Image + if isinstance(image, str): + if os.path.isfile(image): + pil_image = PIL.Image.open(image) + else: + raise ValueError( + f"Incorrect path or URL. URLs must start with `http://` or `https://`, and {image} is not a valid path." + ) + elif isinstance(image, PIL.Image.Image): + pil_image = image + else: + raise ValueError( + "Incorrect format used for the image. Should be a URL linking to an image, a local path, or a PIL image." + ) + + # Handle EXIF orientation + pil_image = PIL.ImageOps.exif_transpose(pil_image) + + # Custom conversion or default to RGB + if convert_method is not None: + pil_image = convert_method(pil_image) + else: + pil_image = pil_image.convert("RGB") + + # Convert to numpy then to tensor; ensure contiguous and copy memory to avoid referencing PIL buffer + np_img = np.asarray(pil_image, dtype=np.float32) # H x W x C, float32 + # Expand channel if grayscale + if np_img.ndim == 2: + np_img = np_img[:, :, None] + if np_img.shape[2] == 4: + # RGBA -> RGB (simply strip alpha); customize convert_method for alpha compositing + np_img = np_img[:, :, :3] + + # Normalize to [0,1] + np_img = np_img / 255.0 + + tensor = torch.from_numpy(np_img).permute(2, 0, 1).contiguous() # C x H x W + + # dtype & device + target_dtype = dtype or torch.float32 + tensor = tensor.to(dtype=target_dtype) + + # Clone to ensure independent memory (optional but safe) + return tensor.clone() + +def base64_to_image(base64_string, output_path): + """ + Save a Base64 string as an image file. + + Args: + base64_string: Base64-encoded string + output_path: Output image path (e.g. 'output.jpg', 'output.png') + """ + try: + # Decode Base64 string + image_data = base64.b64decode(base64_string) + + # Save to file + with open(output_path, 'wb') as f: + f.write(image_data) + + print(f"Image saved to: {output_path}") + return output_path + + except Exception as e: + print(f"Conversion failed: {e}") + return None + +def image_to_base64(img: Union[str, PIL.Image.Image]) -> str: + """ + Convert an image file path or PIL.Image to a Base64 string. + """ + if isinstance(img, str): + pil_image = PIL.Image.open(img) + elif isinstance(img, PIL.Image.Image): + pil_image = img + else: + raise ValueError("img must be a file path or PIL.Image object") + + # Convert to RGB + pil_image = PIL.ImageOps.exif_transpose(pil_image) + pil_image = pil_image.convert("RGB") + + # Save to in-memory buffer + buf = io.BytesIO() + pil_image.save(buf, format="PNG") + byte_data = buf.getvalue() + b64_str = base64.b64encode(byte_data).decode("utf-8") + return b64_str + +def create_tensor(data, name, datatype="FP32"): + """ + Construct a dictionary conforming to the proto::Tensor format (4 top-level fields). + + Args: + data: numpy array or Python list, tensor data + name: tensor name (corresponds to the Proto name field) + datatype: data type (corresponds to the Proto datatype field), default FP32 + + Returns: + dict: dictionary fully matching the proto::Tensor structure + """ + # Convert to numpy array to get shape + if not isinstance(data, np.ndarray): + data = np.array(data) + + # 1. Handle shape: ensure all dimensions are positive (avoid backend invalid dimension errors) + shape = list(data.shape) + print(shape) + if any(dim <= 0 for dim in shape): + raise ValueError(f"Tensor {name} has non-positive dimensions: {shape}, all must be positive") + + # 2. Handle data: flatten and store in the corresponding typed contents field + contents = {} + flat_data = data.flatten().tolist() + if datatype == "FP32": + contents["fp32_contents"] = flat_data # Corresponds to Proto TensorContents.fp32_contents + elif datatype == "INT64": + contents["int64_contents"] = flat_data + elif datatype == "BOOL": + contents["bool_contents"] = flat_data + else: + raise ValueError(f"Unsupported data type: {datatype}, only FP32/INT64/BOOL are supported") + + # 3. Return the 4 top-level fields required by Proto (no extra nesting) + return { + "name": name, # Top-level name field + "datatype": datatype, # Top-level datatype field + "shape": shape, # Top-level shape field + "contents": contents # Top-level contents field + } + + +def test_image_generation(pos_embed): + """Test the image generation API (using the corrected Tensor structure)""" + api_base = "http://127.0.0.1:18018" + api_endpoint = f"{api_base}/v1/image/generation" + model_name = "flux2" + try: + # Generate example embedding vectors (shape must match model requirements) + pooled_prompt_embeds = np.random.rand(768).astype(np.float32) # 1D: [768] + prompt_embeds = np.random.rand(2, 768).astype(np.float32) # 2D: [2, 768] + + ip_adapter_image_embeds = np.random.rand(1, 4, 768).astype(np.float32) # 3D: [1,4,768] + latents = np.ones((1, 4, 32, 32), dtype=np.float32) # 4D: [1,4,32,32] (ensure all shape dims are positive) + + # 2. Construct request payload (Tensor structure corrected) + payload = { + "model": model_name, + "input": { + "prompt": "A cat holding a sign that says hello world", + "prompt_2": "", + "negative_prompt": " ", + "negative_prompt_2": "", + "prompt_embed": create_tensor( + pos_embed.to(torch.float32), + name = "prompt_embeds", + datatype="FP32" + ) + }, + "parameters": { + "size": "1024*1024", + "num_inference_steps": 50, # Note: flux-schnell recommends 4 steps, dev recommends 50; 28 may not be optimal + "guidance_scale": 2.5, # Must be consistent with the Python-side setting + "true_cfg_scale": 3.0, + "num_images_per_prompt": 1, + "seed": 42, + "max_sequence_length": 2048 + }, + "user": "test_user", + "service_request_id": f"req-{int(time.time())}" + } + print("python num_inference_steps:", 50) + # 3. Send request + headers = {"Content-Type": "application/json"} + + response = requests.post( + url=api_endpoint, + headers=headers, + data=json.dumps(payload), + timeout=60 * 5 + ) + + response.raise_for_status() + result = response.json() + # 4. Parse response + print(f"API response: {json.dumps(result, indent=2, ensure_ascii=False)}") + if result.get("output") and result["output"].get("results"): + for idx, image_result in enumerate(result["output"]["results"]): + print(f"\nGenerated image {idx + 1}:") + if image_result.get("url"): + print(f"URL: {image_result['url']}") + elif image_result.get("image"): + print(f"Size: {image_result.get('width')}x{image_result.get('height')}") + base64_to_image(image_result['image'], "./result.png") + else: + print(f"Generation failed: {result.get('message', 'No results returned')}") + + except requests.exceptions.RequestException as e: + print(f"Request error: {str(e)}") + except json.JSONDecodeError: + print("Response format error, unable to parse as JSON") + except Exception as e: + print(f"Processing failed: {str(e)}") + +def calculate_dimensions(target_area, ratio): + width = math.sqrt(target_area * ratio) + height = width / ratio + + width = round(width / 32) * 32 + height = round(height / 32) * 32 + + return width, height + +def main(args: argparse.Namespace): + start = time.time() + + tokenizer = AutoTokenizer.from_pretrained("/path/to/flux2/text_encoder/") + messages = [ + {"role": "system", "content": "You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object\nattribution and actions without speculation."}, + {"role": "user", "content": "A cat holding a sign that says hello world"}, + ] + formatted_input = tokenizer.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=False, + ) + # 4. Construct payload with the formatted string as input + payload = { + "model": "text_encoder", + "input": formatted_input, + "encoding_format": "float" + } + + # Send request to the text-encoder + raw_response = requests.post("http://127.0.0.1:18001/v1/embeddings", json=payload) + result = raw_response.json() + + # Parse embeddings + bytes_data = result["data"][0]["mm_embeddings"][0]["embedding"]["contents"]["bytes_contents"] + embed_data = base64.b64decode(bytes_data) + embed_shape = result["data"][0]["mm_embeddings"][0]["embedding"]["shape"] + pos_embed = torch.frombuffer(bytearray(embed_data), dtype=torch.bfloat16).reshape(embed_shape) + + test_image_generation(pos_embed) + end = time.time() + print(f"Elapsed time: {end - start:.2f} seconds") + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="127.0.0.1") + parser.add_argument("--port", type=int, default=18001) + parser.add_argument("--model", type=str, default="text_encoder") + parser.add_argument("--height", type=int, default=1024) + parser.add_argument("--width", type=int, default=1024) + args = parser.parse_args() + main(args) +``` \ No newline at end of file diff --git a/docs/src/content/docs/en/cookbook/diffusion_models/qwen_image/qwen_image.md b/docs/src/content/docs/en/cookbook/diffusion_models/qwen_image/qwen_image.md new file mode 100644 index 0000000000..1568d9f496 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/diffusion_models/qwen_image/qwen_image.md @@ -0,0 +1,8 @@ +--- +title: "Qwen-Image" +description: "CookBook recipe placeholder for Qwen-Image diffusion model inference" +--- + +This section will collect Qwen-Image diffusion model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/cookbook/diffusion_models/wan/wan2_1.md b/docs/src/content/docs/en/cookbook/diffusion_models/wan/wan2_1.md new file mode 100644 index 0000000000..0a0c148259 --- /dev/null +++ b/docs/src/content/docs/en/cookbook/diffusion_models/wan/wan2_1.md @@ -0,0 +1,8 @@ +--- +title: "Wan2.1" +description: "CookBook recipe placeholder for Wan2.1 diffusion model inference" +--- + +This section will collect Wan2.1 diffusion model inference recipes for xLLM. + +Content will be added as the recipes are organized. diff --git a/docs/src/content/docs/en/design/generative_recommendation_design.md b/docs/src/content/docs/en/design/generative_recommendation_design.md new file mode 100644 index 0000000000..83f6869bf0 --- /dev/null +++ b/docs/src/content/docs/en/design/generative_recommendation_design.md @@ -0,0 +1,1066 @@ +--- +title: "Generative Recommendation Design Document" +sidebar: + order: 2 +--- +## Overview + +xLLM provides generative recommendation inference through the `backend=rec` path. The goal is not to replace the existing recommendation system, but to reuse the LLM inference engine in the recommendation pipeline while keeping the original predictor-side sparse feature processing and online serving capabilities. In practice, the LLM body is executed by xLLM, while the traditional recommendation system continues to handle feature preparation and online integration. + +This document focuses on the following topics: + +- the goal and constraints of generative recommendation inference +- model structure and integration architecture +- why the REC path prefers fixed scheduling and whole-graph multi-step execution +- how `xAttention` and `beam search` cooperate around memory efficiency and execution efficiency +- where the core REC-related code is located in the current branch + +The design goals of this document are: + +- explain the `backend=rec` path with a unified view +- clarify the relation among fixed scheduling, multi-step execution, and custom kernels +- provide a stable base document for future technical sharing and code walkthroughs + +The non-goals of this document are: + +- full training details of recommendation models +- all online serving differences across business scenarios +- replacing detailed module-level API documents + +## 1. Background and Problem + +In recent years, LLM-based generative recommendation has made substantial progress. xLLM has also introduced support for recommendation inference. The goal of generative recommendation is not simply to attach an LLM to a recommendation system, but to use generative modeling to improve candidate expansion and ranking quality, especially metrics such as `CTR`. + +In the current solution, xLLM serves as the unified inference engine and is integrated into the existing prediction pipeline through a shared library (`.so`) interface: + +- the `predictor` side continues to handle sparse feature processing, sample construction, and online service integration; +- the `xLLM` side is responsible for the LLM-related inference computation. + +This split allows the original recommendation engineering capabilities to remain intact, while reusing xLLM's infrastructure in operators, KV cache management, multi-backend execution, and scheduling. + +However, generative recommendation and general LLM inference optimize for different targets. + +- General LLM inference cares more about token-by-token interaction quality, such as returning the first result quickly, reducing the interval between generated tokens, and allowing requests to be inserted or ended dynamically. +- Generative recommendation cares more about total request latency and obtaining better candidate results within a limited number of decoding rounds. + +The reason is straightforward: recommendation is usually not about generating an open-ended passage, but about expanding candidates, comparing candidates, and outputting the best result within a fixed number of rounds. + +![Generative recommendation overview](../../assets/generative_recommendation_overview.png) + +This path often uses `beam search`. In this context, beam search means that at each decoding round the system keeps multiple high-scoring candidate branches instead of only the best one, then keeps expanding and comparing them in later rounds. In recommendation, the purpose is not to generate longer text, but to cover more high-quality candidates within a small number of decoding rounds and improve the final recommendation quality. + +![Beam search in generative recommendation](../../assets/generative_recommendation_beam_search.png) + +Therefore, generative recommendation naturally has two characteristics: + +- fixed-step decoding; +- synchronized comparison of multiple candidates. + +In other words, the real optimization target is not “make one sequence finish earlier”, but “push multiple candidates forward stably in a small number of rounds and compare them efficiently at each round”. This leads directly to the later design choice: fixed scheduling at the control layer and whole-graph multi-step execution at the execution layer, followed by custom operator optimization on top of that stable execution shape. + +### 1.1 Workload characteristics: why GR is not just “another attention model” + +At a high level, generative recommendation still uses attention-based architectures, so it is easy to assume that its serving path can simply reuse the general LLM serving mindset. But the workload profile is very different. + +Generative recommendation typically has the following characteristics: + +- the prompt is long because it carries user history, context, and recommendation-side signals; +- the output is short because the system only needs a fixed-length item token sequence; +- the number of decode rounds is fixed and usually small; +- each decode round is still expensive because candidate expansion is often combined with a large `beam_width` and `top_k`. + +This forms a sharp contrast with general LLM inference: + +- general LLM inference is usually “short prompt + long output”; +- generative recommendation is much closer to “long prompt + short output”. + +So recommendation does not become cheap simply because the output is short. The decode phase is still expensive, and in many cases the expensive part is no longer only attention itself, but the system cost around candidate expansion and beam comparison. + +### 1.2 Three core challenges of this workload + +From the workload characteristics of generative recommendation itself, three challenge categories stand out clearly when compared with the general LLM inference path. + +#### Challenge 1: long-prompt / short-output does not mean decode is cheap + +Even though the decode length is fixed and small, each decode round may still be expensive. Shared-prefix reuse, repeated KV access across beams, and beam-related block movement all become more visible because the system is not amortizing them over a long free-form generation. + +#### Challenge 2: beam search is not only an algorithmic issue + +In generative recommendation, beam search is not just a decoding technique for “better text quality”. It is part of the recommendation search process itself. Once `beam_width` and `top_k` increase, sorting, filtering, valid-item checking, candidate retention, and data-structure reuse all become system-level concerns. + +#### Challenge 3: the bottleneck is also in host-device cooperation + +The system usually runs under strict online latency constraints and high concurrency. If the host still comes back at every step to decide whether to continue, prepare the next input, and resend control to the device, the host-side control path itself becomes a major part of the latency budget. + +### 1.3 The role of this design document + +This document does not try to reproduce an external paper or replace an experiment section. Its role is more practical: it brings the workload characteristics, the system-level problems, and the concrete implementation paths of the current branch into one place, so that later talks, reviews, and code walkthroughs can all use the same technical base. + +## 2. Inference Architecture + +### 2.1 Model Structure + +Generative recommendation has become one of the most important paradigm shifts in modern recommendation systems. It breaks the traditional recall-ranking-rerank cascade and pushes the task from discriminative matching toward generative prediction. This document focuses on two model families that have shown strong quality and have been deployed at scale: OneRec for recall and OneTrans for ranking. + +![OneRec model structure](../../assets/generative_recommendation_model_onerec.png) + +![OneTrans model structure](../../assets/generative_recommendation_model_onetrans.png) + +A shared pattern across these models is that they keep the traditional recommendation signals such as user sequence features, user static features, and context features, then use an input adaptation layer to map heterogeneous recommendation inputs (discrete IDs, continuous values, sequences, and multimodal content) into embeddings that can be consumed by the LLM decoder. The model body itself is still an Encoder+Decoder or Decoder-only LLM structure, which means different parts of the model should be handled by different inference engines. + +### 2.2 Inference Integration Architecture + +Based on the model structure, the current solution splits the model into two groups: + +- the input adaptation layer still belongs to the traditional CTR-style inference domain and is handled by the `predictor` side; +- the LLM body is handled by xLLM. + +As the core LLM inference engine, xLLM provides two integration modes for generative recommendation: RPC integration and shared library (`.so`) integration. + +#### 2.2.1 RPC Integration + +The current marketing and online recall scenarios mainly use the RPC-based integration mode. Its advantage is a clean service boundary, while the downside is the extra RPC overhead. + +#### 2.2.2 Shared Library Integration + +Another mode is to embed xLLM into the predictor side as an internal inference engine for the LLM subgraph. This avoids RPC round trips and is more suitable for future low-latency scenarios. + +![Generative recommendation integration architecture](../../assets/generative_recommendation_integration_architecture_en.svg) + +## 3. Fixed Scheduling and Graph-style Execution + +### 3.1 Fixed-Step Scheduling + +![Background of Orca continuous batching](../../assets/fixed_steps_scheduler_orca.png) + +The figure above comes from the paper *Orca: A Distributed Serving System for Transformer-Based Generative Models*. It explains why continuous batching is useful for general text generation: it avoids idle compute caused by rigid fixed-batch execution. + +Generative recommendation changes that assumption because the decoding length is fixed and the candidate set needs to move forward synchronously. As a result, the REC path is more suitable for `fixed_steps_scheduler` than for continuous batching. The reason is not simply “fixed rounds imply fixed scheduling”. The deeper reason is that the workload itself is organized as a fixed number of rounds. When requests usually finish in a predefined number of rounds and multiple candidate branches must move forward together, the scheduler should focus on sending one stable candidate group efficiently instead of inserting and removing requests at every step. + +The first benefit of `fixed_steps_scheduler` is that it matches `beam search` better. In the decode stage, `beam width` is often large and multiple beams need to move and be compared in the same round. If continuous scheduling is used, every step may trigger batch rebuilding, sequence compaction, index remapping, and state pruning. Those operations make sense for general LLM inference because requests really do end dynamically. In generative recommendation, however, they are often additional cost instead of real value. Under fixed scheduling, the beam group of one request can move forward together inside one fixed window, without repeated batch rebuilding and repeated pruning decisions. + +The second benefit is execution stability. Once the number of rounds, the beam-group size, and the advancing rhythm all become stable, many later optimizations become possible. Buffers can be allocated early, workspace can be reused more easily, and cache access patterns become more regular. This stability makes profiling and capacity planning easier and allows the execution path to be made much more stable. + +![PagedAttention background and fixed-step comparison](../../assets/paged_attention_comparison.png) + +The third benefit is that it reduces overhead outside the real model computation. In recommendation inference, the major cost should belong to candidate expansion, attention computation, and beam comparison. But if the scheduler keeps participating in sequence reordering, batch rebuilding, metadata refresh, and index movement at every step, then extra overhead is introduced even though it is not part of the actual operator work. In this sense, fixed scheduling trades stronger execution determinism for higher throughput, lower scheduling overhead, and more stable runtime behavior. + +Of course, fixed scheduling also has a clear cost: a new request may wait longer. Under continuous scheduling, a new request may have a chance to be inserted after only one step. Under fixed scheduling, it often has to wait until the current fixed window finishes. This leads to a more visible queueing delay. The mitigation direction is not to go back to continuous scheduling, but to introduce `multi-stream` execution. The idea is to decouple the large request groups already running inside a fixed window from newly arriving small request groups and place them on different streams or execution channels. The purpose is not to eliminate waiting entirely, but to keep the throughput advantage of fixed scheduling while reducing the extra access latency of new requests. + +### 3.2 Graph-style Multi-Step Execution + +On top of fixed scheduling, `multi_step_pipeline` becomes the natural execution-side companion. It solves an execution-efficiency problem. Once we know the workload itself always runs a fixed number of rounds and does not usually end early, then there is no need to involve the host at every step. There is no need to perform a `D2H` synchronization every round just to ask whether the batch is finished, and there is no need to perform another `H2D` transfer every round just to prepare the next round's input. A more efficient design is to prepare the space, indices, and data structures needed by later rounds at the first step, and then let the device continue advancing through the later rounds. + +This brings several direct benefits: + +- fewer `D2H/H2D` round trips and less host participation; +- lower launch and control overhead at every round; +- better device-side data reuse because more intermediate data stays on device; +- a more pipeline-like execution flow instead of a repeated stop-prepare-run cycle. + +For a fixed-round recommendation workload, this is clearly more efficient than returning to the host after every step. + +Another important but often underestimated benefit of `multi_step_pipeline` is that it creates a better execution environment for custom operators. This is the point where `xAttention` and `beam search` custom kernels can be discussed together. `fixed step` solves scheduling stability, while whole-graph multi-step execution plus custom kernels solves execution efficiency. + +## 4. Memory Management and Operator Co-optimization + +### 4.1 Compute and Memory Bottlenecks + +#### 4.1.1 Model Input and Output Characteristics + +In the current generative recommendation setup, an item ID is represented by a fixed-length token sequence. As a result, `decode_step` is a known small constant, for example 3. One request can be summarized as: + +- one prefill stage with a long user-history context; +- `decode_step` rounds of decode, where each round generates one token and the final token sequence is combined into an item ID. + +Even if the number of decode rounds is small, the per-step cost is not small. To improve recall and diversity, generative recommendation often uses a large `beam_width`. In addition, each beam may expand to `top_k` candidates, and the system then selects the new beam set from a global candidate pool of `beam_width × top_k`. For example, when `beam_width=512` and `top_k=512`, the candidate pool size of one decode round reaches 262144 (about 2.6×10^5). So although the number of rounds is limited, the search and KV access cost per step is still considerable. + +#### 4.1.2 Storage Redundancy and Memory Fragmentation + +The main bottlenecks of this inference service can be summarized into two groups, and `xAttention` is designed around both of them. + +The first is redundant bandwidth consumption in attention. Shared prefixes are not explicitly represented as reusable structures. When beam width is large, all beams share the same long prompt, but a generic implementation often organizes KV as if every beam were a full independent sequence. This causes shared KV to be loaded repeatedly along the beam dimension and reduces the effective arithmetic intensity of the attention kernel, eventually making the path bandwidth-bound. + +The second is KV cache copying and fragmentation. Beam search frequently forks and retires branches, which leads to beam reordering. For a block-based KV management scheme such as PagedAttention, “reordering + block alignment” often implies block copying, fragmentation, and extra memory waste. Both memory capacity and memory bandwidth get amplified in the wrong direction. + +### 4.2 `xAttention` Design Principle + +#### 4.2.1 KV Cache Layout Optimization + +![xAttention KV cache layout](../../assets/xattention_kv_layout_en.svg) + +Given the fixed structure of generative recommendation inference, xAttention redesigns both KV cache organization and the attention execution strategy. The shared prefix is stored only once at the physical-memory level, while beam branching and reordering no longer cause high-cost copying. + +The first key idea is to split KV cache into two groups: + +- **Shared KV**: prompt KV generated in prefill and shared by all beams; +- **Unshared KV**: newly generated KV in decode, managed at token granularity for each beam. + +Once the cache is split this way, Unshared KV only stores the decode-generated tokens, which avoids both block copying and unnecessary memory waste. + +#### 4.2.2 Attention Compute Optimization + +![xAttention three-stage execution](../../assets/xattention_three_stage_pipeline_en.svg) + +To avoid concatenating Shared KV and Unshared KV into one long logical sequence, xAttention splits one attention computation into three stages: + +1. **shared stage**: compute local softmax statistics and partial outputs only on Shared KV; +2. **unshared stage**: compute local statistics and partial outputs only on Unshared KV; +3. **merge stage**: use OnlineSoftmax to merge the two parts stably. + +At the parallelization level, shared, unshared, and merge are assigned to different execution units or queues to form a pipeline. The goal is to overlap Shared and Unshared computation as much as possible while minimizing synchronization points. + +### 4.3 Treating beam search as a system problem + +If `xAttention` addresses the question “how can attention reuse shared context and avoid wasteful KV movement”, then this section addresses a different but equally important question: “how can beam search avoid turning recommendation decode into a sorting-heavy control bottleneck”. + +From a systems perspective, beam search in recommendation carries several cost layers at once: + +- each round must select a new beam set from a large candidate pool; +- not every token combination corresponds to a valid item; +- old candidates are discarded while new candidates are constantly introduced; +- if the implementation rebuilds data structures and performs full sorting every round, the overhead grows quickly. + +So this part should be understood less as “one sorting kernel” and more as a system-level optimization strategy around beam search. Its goals include: + +- terminate unnecessary sorting work as early as possible; +- filter invalid item paths as early as possible; +- reuse data structures across rounds instead of reconstructing them repeatedly. + +For a technical talk, the key message here is that beam search is not an accessory cost in recommendation serving. It is part of the main decode cost, and therefore has to be designed together with scheduling, memory layout, and hot operator paths. + +### 4.4 A system-level view of overlap and parallelism + +The third challenge is not that one operator is slow, but that the whole serving pipeline is not naturally shaped for the recommendation workload. In this document, this refers to the system-level effort to maximize overlap across host-side scheduling, engine-side execution, worker-side multi-round progression, and stream-level parallelism. + +At least three ideas belong to this layer: + +1. **clearer host-device role split** + - the host should participate less in every round; + - the device should carry more of the fixed-round progression directly. + +2. **more pipeline-friendly execution** + - while the current round is being executed, next-round inputs should already be under preparation; + - scheduler, batch builder, and worker should avoid introducing unnecessary idle boundaries. + +3. **multi-stream and multi-pipeline concurrency** + - a fixed execution window naturally increases waiting cost for new requests; + - but that cost can be partially offset through multiple streams or multiple execution pipelines. + +So, in the context of this document, this section is best understood as the system layer that combines fixed-step stability, device-side multi-step progression, and multi-stream overlap into one serving strategy. + +### 4.5 Why this design direction is worth the effort + +From the workload characteristics of generative recommendation, treating it as a special serving path is not over-engineering. It has clear system-level value. + +The reason is that this workload combines several properties that do not usually appear together in the general LLM path: + +- the input is long while the output is short; +- the decode rounds are fixed, but one decode round is still expensive; +- both `beam_width` and candidate-pool size are non-trivial; +- host-side control, batch rebuilding, and data movement can easily dominate the latency budget. + +As a result, simply reusing the general path tends to preserve unnecessary system cost. Once fixed-step scheduling, multi-step execution, shared/unshared KV design, beam search handling, and multi-stream overlap are considered together, the gain can appear at multiple levels at once: lower scheduling cost, less host participation, a cleaner memory layout, and a more stable hot path for custom operators. + +This also gives the technical talk a strong closing point for this section: the reason to separate recommendation from the general LLM path is not just conceptual neatness, but the fact that recommendation decode really creates a different serving problem, and the payoff of handling it explicitly can be significant. + +## 5. Code Layout + +The current branch organizes the generative recommendation path as follows. + +External integration: +- `xllm/c_api/rec.h` +- `xllm/c_api/internal/rec.cpp` +- `xllm/c_api/examples/simple_rec_completions.cpp` + +Service entry: +- `xllm/api_service/rec_completion_service_impl.cpp` +- `xllm/api_service/chat_service_impl.cpp` +- `xllm/api_service/api_service.cpp` +- `xllm/api_service/api_service.h` + +Scheduling and engine: +- `xllm/core/distributed_runtime/rec_master.cpp` +- `xllm/core/distributed_runtime/rec_master.h` +- `xllm/core/scheduler/fixed_steps_scheduler.cpp` +- `xllm/core/scheduler/fixed_steps_scheduler.h` +- `xllm/core/distributed_runtime/rec_engine.cpp` +- `xllm/core/distributed_runtime/rec_engine.h` + +Batch / request / proto: +- `xllm/core/framework/batch/rec_batch_input_builder.cpp` +- `xllm/core/framework/batch/rec_batch_input_builder.h` +- `xllm/core/framework/batch/rec_multi_round_batch_input_builder.cpp` +- `xllm/core/framework/batch/rec_multi_round_batch_input_builder.h` +- `xllm/core/framework/request/rec_type.h` +- `xllm/proto/rec.proto` +- `xllm/proto/completion.proto` +- `xllm/proto/xllm_service.proto` + +Runtime / worker: +- `xllm/core/runtime/rec_worker_impl.cpp` +- `xllm/core/runtime/rec_worker_impl.h` + +Kernel hot path: +- `xllm/core/layers/cuda/xattention.cpp` +- `xllm/core/layers/cuda/flashinfer_attention.cpp` +- `xllm/core/kernels/cuda/xattention/beam_search.cpp` +- `xllm/core/kernels/cuda/xattention/cache_select.cu` + +## 6. Current-branch Execution Flow + +To align the design with the actual implementation, the current branch can be understood as the following execution chain: + +1. **External entry** + - In shared-library mode, requests enter from `xllm/c_api/internal/rec.cpp` through `xllm_rec_text_completions`, `xllm_rec_token_completions`, or `xllm_rec_chat_completions`. + - In service mode, requests enter from `xllm/api_service/rec_completion_service_impl.cpp` or `chat_service_impl.cpp`, and are then forwarded to `RecMaster`. + +2. **Request convergence in `RecMaster`** + - `RecMaster` unifies different request forms such as prompt input, token input, and raw embedding input. + - It also distinguishes `kOneRec` and `kLlmRec`, then selects the corresponding request pipeline. + +3. **Entering fixed scheduling** + - `RecMaster` creates `FixedStepsScheduler` directly during initialization. + - Instead of rebuilding decode batches dynamically at every step, the scheduler is centered on a fixed number of rounds and a stable candidate group. + +4. **Engine execution** + - `RecEngine` then selects an execution path according to `RecPipelineType`. + - For the `LlmRec` multi-round scenario, execution is routed into `RecMultiRoundEnginePipeline`, which pushes more decode control logic down toward the worker side. + +5. **Batch building and input construction** + - `RecBatchInputBuilder` and `RecMultiRoundBatchInputBuilder` organize sequences, step information, decode positions, sampling parameters, and other metadata into `ForwardInput`. + - `step_meta` is especially important here because it provides the per-round information needed for multi-step execution. + +6. **Multi-round execution inside the worker** + - `RecWorkerImpl::LlmRecMultiRoundPipeline::step()` performs a device-side round loop. + - For each round, it coordinates: + - current-round input preparation + - model forward + - sample processing + - beam search + - cache selection + - preparation for the next round + +7. **Hot operator path** + - Attention-related execution goes through `xattention.cpp` and `flashinfer_attention.cpp` + - Beam-related selection goes through `beam_search.cpp` + - Cache selection after beam reordering goes through `cache_select.cu` + +From a technical-sharing perspective, this chain is a useful narrative backbone because it connects fixed scheduling, whole-graph multi-step execution, and custom kernels into one coherent execution story instead of presenting them as isolated optimizations. + +## 7. Trade-offs and Applicability Boundaries + +This design does not mean that fixed scheduling is always better than continuous scheduling, nor does it mean that multi-step pipeline execution fits every generation task. It works well because the current generative recommendation workload has a few specific characteristics: + +- the number of decode rounds is relatively fixed and requests usually do not terminate early like open-ended text generation; +- one request often carries a large `beam_width`, and multiple beams must be compared synchronously; +- total request latency matters more than token-by-token interactivity; +- device-side state such as KV cache, positions, and beam tensors can be prepared and reused in a relatively stable way. + +When those conditions hold, fixed scheduling and multi-step execution provide clear benefits. At the same time, they also have clear boundaries: + +### 7.1 Boundary of fixed scheduling + +- If request lengths vary dramatically and a large number of requests end early, continuous scheduling regains its advantage. +- If the business priority is to insert new requests as soon as possible, fixed scheduling naturally becomes less attractive. +- If candidate expansion no longer depends on synchronized beam progression, the benefit of a fixed execution window decreases. + +### 7.2 Boundary of `multi_step_pipeline` + +- If each step still requires strong host-side decisions, the value of device-side multi-step execution becomes smaller. +- If shape, batch layout, or key inputs change significantly at every round, it becomes much harder to keep a stable whole-graph execution path. +- If backend operators themselves are not ready for stable multi-round device-side execution, whole-graph execution will remain only a conceptual idea. + +### 7.3 Boundary of custom operators + +`xAttention` and custom `beam search` kernels are valuable because the execution shape is already stable enough. Without fixed-step scheduling and multi-step device-side progression, many operator-level optimizations would be diluted by repeated data movement, batch rebuilding, and extra host participation. + +So the more accurate order of reasoning is: + +1. first confirm that the workload truly fits fixed scheduling; +2. then confirm that multi-round execution can be pushed down to the device side; +3. only then optimize the stable hot path with custom operators. + +This is what makes `fixed_steps_scheduler`, `multi_step_pipeline`, `xAttention`, and `beam search` a coherent design stack rather than four unrelated optimization tricks. + +## 8. Code Path Appendix + +The purpose of this appendix is not to repeat the file list in the previous section, but to provide a practical reading order. If someone later wants to expand this document, prepare a technical talk, or walk through the implementation, the following path is a more useful starting point. + +### 8.1 Start from the external entry + +If the goal is to understand how `predictor` or the shared-library integration enters xLLM, start with: + +- `xllm/c_api/rec.h` + - exposes `xllm_rec_create`, `xllm_rec_initialize`, `xllm_rec_text_completions`, `xllm_rec_token_completions`, and `xllm_rec_chat_completions` + - useful for understanding what the REC-facing external contract looks like +- `xllm/c_api/internal/rec.cpp` + - the actual CAPI implementation + - useful for seeing how request parameters are wrapped and forwarded in `.so` mode +- `xllm/c_api/examples/simple_rec_completions.cpp` + - the smallest runnable example + - useful when explaining what a dynamic-library integration looks like in practice + +If the technical-sharing version needs one “minimal usage example”, this is the best layer to cite first. + +### 8.2 Then read the service entry layer + +If the focus is on RPC mode or the unified service path, continue with: + +- `xllm/api_service/api_service.cpp` + - dispatches service implementations according to `FLAGS_backend` + - under `backend == "rec"`, both `rec_completion_service_impl_` and `chat_service_impl_` are attached +- `xllm/api_service/rec_completion_service_impl.cpp` + - forwards REC completion requests into `RecMaster` + - this is where `routing`, `input_tensors`, and `RequestParams` are assembled together +- `xllm/api_service/chat_service_impl.cpp` + - also provides a chat entry for `RecMaster` + - useful to show that REC is not limited to token completion only + +This layer is useful in a technical talk when explaining that `backend=rec` is not a side path outside the service framework, but a first-class backend integrated into the existing service entry. + +### 8.3 Then read scheduling and engine as one chain + +If the main story is “why fixed step is a better fit for REC”, a practical reading order is: + +1. `xllm/core/distributed_runtime/rec_master.h` +2. `xllm/core/distributed_runtime/rec_master.cpp` +3. `xllm/core/scheduler/fixed_steps_scheduler.h` +4. `xllm/core/scheduler/fixed_steps_scheduler.cpp` +5. `xllm/core/distributed_runtime/rec_engine.h` +6. `xllm/core/distributed_runtime/rec_engine.cpp` + +The chain can be understood as: + +```text +Rec request + -> RecMaster + -> FixedStepsScheduler + -> RecEngine + -> RecEnginePipeline + -> Worker / worker_clients +``` + +The most important takeaways in this layer are: + +- `RecMaster` converges multiple request forms and chooses the request pipeline +- `FixedStepsScheduler` organizes requests into a batch shape that matches fixed-round progression +- `RecEngine` bridges the scheduler output to the actual execution path +- `RecMultiRoundEnginePipeline` is the concrete code path that represents “push multi-round decode control further down” + +If the talk needs to prove that “fixed step is not just an idea but the actual code path”, this is the core evidence layer. + +### 8.4 Read batch builders to understand why multi-step execution is possible + +To explain why `multi_step_pipeline` is possible, it is not enough to stop at the scheduler. The batch builder layer is equally important: + +- `xllm/core/framework/batch/rec_batch_input_builder.h` +- `xllm/core/framework/batch/rec_batch_input_builder.cpp` +- `xllm/core/framework/batch/rec_multi_round_batch_input_builder.h` +- `xllm/core/framework/batch/rec_multi_round_batch_input_builder.cpp` +- `xllm/core/framework/batch/batch.cpp` + +The key points here are: + +- `RecBatchInputBuilder::create(...)` chooses different builders according to `RecType` and multi-round mode +- `RecMultiRoundBatchInputBuilder` is not just a small variation of the default builder, but a dedicated implementation for multi-round decode input construction +- `step_meta`, `decode_positions`, `sampling params`, and `batch forward type` are assembled here before being sent further into runtime + +So if the document or talk wants to explain “why later rounds can already be prepared at the first step”, this layer is more important than only talking about the engine loop. + +### 8.5 Read worker-side multi-round execution next + +The most valuable implementation details of `multi_step_pipeline` are inside: + +- `xllm/core/runtime/rec_worker_impl.h` +- `xllm/core/runtime/rec_worker_impl.cpp` + +Especially these parts: + +- `RecWorkerImpl::step_async(...)` + - shows how work is dispatched into the worker and onto the target stream +- `RecWorkerImpl::LlmRecMultiRoundPipeline::prepare_inputs(...)` + - shows how multi-round inputs enter runtime +- `RecWorkerImpl::LlmRecMultiRoundPipeline::allocate_kv_caches_related()` + - shows why fixed-step execution is friendly to early KV-related allocation +- `RecWorkerImpl::LlmRecMultiRoundPipeline::step(...)` + - this is the single most important function for multi-round execution + - it makes the relationship among round loop, beam search, cache select, and next-round preparation explicit +- `compute_next_round_input_async(...)` + - this is the key function when explaining why host round-trips can be reduced + +If a technical talk wants to emphasize execution efficiency rather than scheduling policy, this is the best layer to expand. + +### 8.6 Finally read the hot operator path + +If the goal is to explain why `xAttention` and custom `beam search` kernels matter, the recommended reading order is: + +- `xllm/core/layers/cuda/xattention.cpp` +- `xllm/core/layers/cuda/flashinfer_attention.cpp` +- `xllm/core/kernels/cuda/xattention/xattention_ops_api.h` +- `xllm/core/kernels/cuda/xattention/beam_search.cpp` +- `xllm/core/kernels/cuda/xattention/cache_select.cu` + +This layer helps answer three questions: + +1. where the stable attention execution path actually lives +2. why beam search is not only a scheduling topic but also a hot kernel path +3. why cache selection after beam reordering must be discussed together with execution shape + +In other words, if the technical-sharing narrative wants to connect `fixed_steps_scheduler`, `multi_step_pipeline`, `xAttention`, and `beam search` into one coherent story, it eventually has to land here. + +### 8.7 Recommended reading order + +If this document is expanded further later, the recommended code-reading order is: + +```text +entry + -> api_service + -> RecMaster + -> FixedStepsScheduler + -> RecEngine + -> RecBatchInputBuilder / RecMultiRoundBatchInputBuilder + -> RecWorkerImpl::LlmRecMultiRoundPipeline + -> xAttention / beam_search / cache_select +``` + +The advantage of this order is that it matches how people usually understand the system: + +- first, how the request enters the system +- then, why REC prefers fixed-step scheduling +- then, how multi-step execution is actually pushed down to the device side +- finally, why custom operators become valuable only after the execution shape becomes stable + +This makes the logic easier to present in a technical talk and easier to reuse in future documentation work. + +## 9. Key Code Anchor Index + +If this design document is later extended into a technical talk, a PR explanation, or a deeper code walkthrough, the following anchors are the fastest way to connect the document back to the current branch implementation. + +### 9.1 Entry and service-layer anchors + +- `xllm/core/distributed_runtime/rec_master.cpp:575` + - `RecMaster::handle_request(...)` + - prompt / prompt_tokens / input_tensors entry +- `xllm/core/distributed_runtime/rec_master.cpp:603` + - `RecMaster::handle_request(...)` + - chat-message entry +- `xllm/core/distributed_runtime/rec_master.cpp:651` + - `RecMaster::handle_request(const std::vector& prompt_tokens, ...)` + - token / raw-input style entry + +This group is useful when the talk needs to answer a simple question first: how exactly does a REC request get converged into `RecMaster`. + +### 9.2 Scheduling anchors + +- `xllm/core/scheduler/fixed_steps_scheduler.cpp:337` + - `FixedStepsScheduler::step(const absl::Duration& timeout)` + - the actual scheduling step that advances the fixed-step path +- `xllm/core/scheduler/fixed_steps_scheduler.cpp:186` + - `FixedStepsScheduler::prepare_batch()` + - useful for explaining how requests are grouped under fixed scheduling +- `xllm/core/framework/batch/rec_batch_input_builder.cpp:29` + - `RecBatchInputBuilder::create(...)` + - useful for explaining how builders are selected according to `RecType` and multi-round mode + +Taken together, these anchors are strong evidence that fixed-step scheduling is not just a conceptual preference but the actual scheduling choice in the code path. + +### 9.3 Engine and multi-round execution anchors + +- `xllm/core/distributed_runtime/rec_engine.cpp:901` + - `RecEngine::RecMultiRoundEnginePipeline::step(...)` + - shows how engine-side execution is pushed into the multi-round pipeline +- `xllm/core/runtime/rec_worker_impl.cpp:849` + - `RecWorkerImpl::LlmRecMultiRoundPipeline::step(...)` + - the single most important multi-round execution function on the worker side +- `xllm/core/runtime/rec_worker_impl.cpp:1011` + - call site of `xllm::kernel::cuda::beam_search(...)` +- `xllm/core/runtime/rec_worker_impl.cpp:1066` + - call site of `xllm::kernel::cuda::cache_select(...)` + +If the talk needs to make it explicit that beam search and cache select are not only “scheduling concepts” but hot device-side execution paths, this is the right group of anchors to cite. + +### 9.4 How to use this anchor list + +This list does not need to be repeated in the main narrative, but it is especially useful as: + +- a “code evidence” slide in a technical talk +- a “key implementation locations” section in a PR description +- a quick response set when a reviewer asks where a specific statement comes from + +If the document is extended further later, these anchors are also the most practical starting points for adding deeper function-level explanations. + +## 10. Suggested Talk Order and Reading Strategy + +The previous sections, especially the code-path appendix and the key-code anchor list, are useful as a reference base. But if this document is later turned into a real technical talk, it also needs a more presentation-friendly reading order. + +### 10.1 Recommended talk order + +If the audience is not directly involved in implementing `backend=rec`, the talk should not start from `RecWorkerImpl` or `beam_search.cpp`. A more effective order is: + +1. **Start from the business goal** + - explain why generative recommendation and general LLM inference optimize for different targets + - explain why recommendation cares more about end-to-end request latency and fixed-round candidate comparison + +2. **Then explain the scheduling choice** + - why `fixed_steps_scheduler` is a better fit for this workload + - why a large `beam_width` makes fixed scheduling more valuable than continuous scheduling + +3. **Then explain the execution model** + - why `multi_step_pipeline` can push multi-round decode control further down to the device side + - why this reduces host round-trips and control overhead + +4. **Then explain custom operators** + - why `xAttention` and `beam search` optimizations become meaningful only after the execution shape becomes stable + - why those two topics should be presented together rather than as isolated tricks + +5. **Finally return to the code** + - use the code path and key anchors to prove the previous conclusions + +This order works well because the audience first understands the design motivation, then the execution strategy, and only after that the implementation evidence. + +### 10.2 For internal code walkthroughs, the order can be reversed + +If the audience already works on xLLM or recommendation infrastructure, the order can be more implementation-driven: + +1. start from `RecMaster -> FixedStepsScheduler -> RecEngine` +2. then `RecBatchInputBuilder` +3. then `RecWorkerImpl::LlmRecMultiRoundPipeline` +4. finally `xAttention / beam_search / cache_select` + +This path is more efficient for an internal walkthrough because it follows the actual call stack. The downside is that it pushes new readers into implementation detail before they have the design context. + +### 10.3 The three takeaways worth remembering + +If the document is later compressed into a short technical presentation, the whole design can be summarized into three lines: + +- `fixed_steps_scheduler` solves scheduling stability; +- `multi_step_pipeline` solves multi-round execution efficiency; +- `xAttention` and `beam search` custom kernels turn that stable execution shape into real performance gains. + +Those three lines are the most important part to remember. The rest of the document can be seen as evidence supporting them. + +## 11. Comparison with the General LLM Inference Path + +To avoid treating `backend=rec` as “just another small variation of LLM inference”, it is useful to compare it explicitly with the general LLM inference path. + +### 11.1 Different optimization targets + +General LLM inference is usually optimized for token-by-token generation experience: return the first result quickly, keep the token interval small, and allow new requests to enter as early as possible. +`backend=rec`, in contrast, is optimized for candidate expansion and comparison within a fixed number of rounds, so the primary concern becomes end-to-end request latency rather than the earliest completion of one sequence. + +That means both paths perform decode, but the target is already different: + +- general LLM inference is more about dynamic request management +- generative recommendation is more about synchronized progression inside a fixed execution window + +### 11.2 Different scheduling focus + +In general LLM inference, continuous scheduling is valuable because: + +- some sequences can end early and free space immediately +- batches can be rebuilt continuously +- dynamic insertion and dynamic exit are frequent and meaningful + +In `backend=rec`, the picture is different: + +- the number of decode rounds is more fixed +- `beam_width` is larger +- multiple candidates under the same request must be compared in the same round +- frequent batch rebuilding tends to amplify scheduling cost rather than reduce it + +So the key difference is not simply “fixed-step vs continuous”, but “stability-first scheduling” versus “flexibility-first scheduling”. + +### 11.3 Different execution model + +General LLM inference often allows the host to continue participating at each decode step, including step completion checks and next-step preparation. +Generative recommendation, because of its fixed-round and synchronized-candidate nature, benefits more from preparing multi-round structures early and letting the device continue forward. + +That is why `multi_step_pipeline` is especially valuable here: it is not only about removing a few memcpy calls, but about replacing a host-driven step-by-step control model with a device-side multi-round progression model. + +### 11.4 Different way to realize operator-level gains + +In general LLM inference, operator optimization often directly improves one decode step or the generic attention path. +In `backend=rec`, the gain of operator optimization depends much more on whether the execution shape has already been stabilized. + +If fixed scheduling is not established and batches are still rebuilt frequently, or if multi-step execution is not established and the host still repeatedly intervenes, a large part of the operator-level gain will be diluted by extra data movement and control overhead. +That is why the more accurate order in recommendation is: + +1. stabilize scheduling first +2. push multi-round execution to the device side +3. optimize the stable hot path with `xAttention`, `beam search`, and `cache_select` + +### 11.5 Design boundaries and applicability conditions + +The purpose of this comparison is not to claim that generative recommendation and general LLM inference are completely unrelated. The real purpose is to make it explicit which parts can be reused from the general path and which parts must be redesigned around the `backend=rec` workload. + +From the analysis above, `fixed_steps_scheduler`, `multi_step_pipeline`, `xAttention`, and `beam search` need to be discussed together because they depend on the same preconditions: + +- output length is fixed or approximately fixed; +- the number of decode rounds is small, but the cost of one round is still high; +- one request maintains a relatively large `beam_width`; +- candidate expansion and candidate comparison are part of the main serving path rather than optional post-processing; +- device-side state can be prepared early and reused across later rounds. + +Only when these conditions hold at the same time do fixed scheduling and whole-graph multi-step execution deliver stable gains. Once these conditions disappear, for example because requests vary dramatically in length, many sequences terminate early, or new-request latency matters more than throughput, the current `backend=rec` design may no longer be the best choice. + +It is also important to note that these four parts are not parallel ideas. They form a layered dependency: + +The first layer is `fixed_steps_scheduler`. It stabilizes the scheduling entrance. If the scheduler still rebuilds batches and prunes sequences at every step, the later execution path cannot become stable. + +The second layer is `multi_step_pipeline`. It stabilizes the progression of multi-round execution. Only after the scheduling window is stable does it become meaningful to push input preparation, KV organization, and round progression further down to the device side. + +The third layer is `xAttention`. It addresses the memory and bandwidth behavior of attention in the recommendation workload. It only pays off consistently when execution itself has already become stable enough. + +The fourth layer is `beam search` optimization. It reduces search and filtering cost under a large candidate space, but because beam logic directly participates in the main decode path, it cannot be treated as an isolated post-processing step. + +In short, the current branch should be understood not as four independent optimizations, but as one layered design shaped by the recommendation workload itself. + +### 11.6 Scenarios where the benefit is limited + +To avoid treating the current design as a universal answer, its boundaries should also be stated explicitly. + +The first category is where fixed scheduling is not ideal. If request lengths vary dramatically, many sequences terminate early, or the system cares more about the earliest insertion of a new request than about total batch throughput, continuous scheduling may become more attractive again. + +The second category is where whole-graph multi-step execution is hard to sustain. If each round still requires strong host-side decisions, or if shape, batch layout, or key inputs change significantly from round to round, then the benefits of `multi_step_pipeline` become weaker. + +The third category is where operator-level gains are limited. If beam size is small, candidate expansion is cheap, or the shared prefix is short, the engineering payoff of `xAttention` and beam-related optimization may still exist, but it will be much smaller than in a typical recommendation-serving workload. + +Writing down these boundaries is useful for two reasons: it prevents readers from assuming unconditional applicability, and it gives reviewers a clearer way to distinguish between design assumptions and implementation defects. + +## 12. Verification and Acceptance Guidance + +Verification should not stop at “the document renders correctly”. It should also answer whether the implementation in the current branch actually supports the design described in this document. + +### 12.1 Document-level verification + +The most basic verification still matters: + +- both the Chinese and English design documents render correctly; +- all image references resolve to actual files under `docs/assets/`; +- the English document points to English diagrams instead of Chinese-labeled figures; +- the entry pages and related feature pages can navigate to this design document. + +This level of verification ensures that the document itself is consumable. + +### 12.2 Code-path consistency verification + +The second level is to verify that the key paths described in the document really exist in the current branch and that their roles match the implementation: + +- whether `RecMaster` truly converges requests and selects the proper pipeline; +- whether `FixedStepsScheduler` is actually the fixed scheduling entry for `backend=rec`; +- whether `RecEngine` is really the engine-side execution organizer; +- whether `RecWorkerImpl::LlmRecMultiRoundPipeline` is really the device-side multi-round execution path; +- whether the paths of `xAttention`, `beam_search`, and `cache_select` match the design description. + +This level prevents the document from becoming a “target architecture note” that no longer matches the branch. + +### 12.3 Verification items tied to design goals + +If this document is used as a stable base for implementation discussion or technical sharing, the verification items should also map directly back to the design goals. + +For fixed scheduling, useful checks include: + +- whether decode still rebuilds batches frequently; +- whether beam-related sequences still trigger heavy pruning and movement every round; +- whether scheduler overhead is actually reduced in profiling. + +For `multi_step_pipeline`, useful checks include: + +- whether the host still decides termination at every round; +- whether next-round inputs can already be prepared while the current round is executing; +- whether `D2H/H2D` round trips are reduced relative to a host-driven round-by-round path. + +For `xAttention`, useful checks include: + +- whether shared KV is really stored once at the physical-storage level; +- whether unshared KV really carries only decode-generated tokens; +- whether repeated loading of the shared prefix is reduced. + +For `beam search`, useful checks include: + +- whether sorting and filtering still dominate the cost under large `beam_width`; +- whether beam-related data structures are reused across rounds; +- whether item filtering and candidate screening are already integrated with the main decode path. + +### 12.4 Final acceptance criteria + +If this document is treated as the design baseline of the current branch, then at minimum it should satisfy the following: + +- document level: bilingual readability, complete references, and reachable navigation; +- structure level: the relationship among scheduling, execution, and custom operators is clearly described; +- code level: key paths, key functions, and key files can be matched to the branch; +- design level: assumptions, boundaries, and trade-offs are stated clearly; +- sharing level: a reader can extract a talk outline directly from the document without rebuilding the logic from scratch. + +Only when those conditions are met does the document become a stable base shared by design discussion, technical sharing, and code review. + +## 13. FAQ / Common Misconceptions + +### Q1: Does fixed-step scheduling always outperform continuous scheduling? + +No. Fixed-step scheduling is a strong fit for the current `backend=rec` workload because the decode rounds are more fixed, `beam_width` is larger, and candidate comparison is more synchronized. If these assumptions disappear, continuous scheduling may become more attractive again. + +### Q2: Is `multi_step_pipeline` only about reducing memcpy calls? + +No. Fewer `D2H/H2D` round trips are only the most visible benefit. The more important change is the control model itself: instead of letting the host drive every round, the system pushes as much multi-round progression as possible down to the device side. + +### Q3: Why are `xAttention` and `beam search` discussed in the same section? + +Because they depend on the same condition: execution shape must already be stable. If the scheduler still rebuilds batches frequently, or the host still intervenes heavily at each round, then a large part of the operator-level gain will be diluted by system overhead. + +### Q4: Is this design only about OneRec? + +No. The document uses recommendation serving as the main narrative and often uses OneRec as an example, but the more important point is the workload shape behind `backend=rec`, not a single model name. + +### Q5: Why does the document repeatedly emphasize “stabilize scheduling first, optimize operators later”? + +Because order matters in system design. If the scheduling entrance is still unstable, and the execution path still depends heavily on host-side intervention, then operator-level optimization will struggle to produce stable end-to-end gains. + +## 14. Contract and Invariants + +If this document is expected to serve as a base for implementation and review, explanation alone is not enough. A few key contracts and invariants should be made explicit. + +### 14.1 External input shapes + +At a high level, the current `backend=rec` path accepts at least three kinds of inputs: + +- **prompt-based input** + - used when recommendation requests enter through text-like prompts +- **token / raw input** + - used when token sequences, embeddings, or related raw structures are already prepared outside +- **chat-style input** + - used when recommendation requests enter through a conversation-like service path + +The key point is not the number of input forms, but the fact that all of them must eventually converge into one unified request state and one scheduler / engine / worker execution chain. + +### 14.2 Key data that directly affects multi-round execution + +For `backend=rec`, the following data items form the most important contract surface: + +- `beam_width` + - controls how many branches are kept after each round +- `top_k` + - controls how many next-token candidates are expanded per branch +- `decode_step` / `total_round` + - defines the fixed-round boundary of execution +- `decode_positions` + - defines how token positions are organized across rounds +- shared / unshared KV + - defines how KV cache is split and reused in decode + +If these data items are not organized early enough and consistently enough, the gains of fixed-step scheduling and multi-step execution will not remain stable. + +### 14.3 Invariants that the document should keep stable + +If this document continues to grow later, the following invariants are worth preserving: + +- whenever the document mentions fixed-step scheduling, it should also explain the boundary conditions; +- whenever it mentions `multi_step_pipeline`, it should state which host-side work is reduced; +- whenever it mentions `xAttention`, it should explain its dependency on KV organization; +- whenever it mentions `beam search`, it should treat it as part of the main serving cost rather than only an algorithmic trick; +- all code anchors must continue to match the current branch and must not silently drift toward future paths. + +## 15. Implementation Status Matrix + +To avoid mixing design intent and branch reality, it is useful to make the current alignment explicit. This is not meant to be an exhaustive implementation matrix, but rather a stable summary of what the current document already treats as grounded facts. + +### 15.1 Capabilities that already have concrete implementation anchors + +- **fixed scheduling** + - the current branch contains `FixedStepsScheduler` + - it can be cited as a concrete scheduling fact rather than a conceptual placeholder +- **multi-round execution path** + - the current branch contains `RecMultiRoundEnginePipeline` + - the worker side contains `LlmRecMultiRoundPipeline` +- **beam search hot path** + - the current branch already contains beam-search-related call paths and kernels +- **cache select hot path** + - the current branch already contains cache-select-related call paths and kernels +- **xAttention / flashinfer path** + - the current branch already contains the relevant implementation files and call chain + +### 15.2 Parts that still remain design-level abstractions in this document + +- the relationship among fixed scheduling, multi-step execution, and custom operators + - this document explains it as a system design + - it is not represented by one single class or one single file +- the end-to-end gain of multi-stream overlap + - the current branch contains the concurrency and stream infrastructure + - but the gain discussion in the document still belongs to system-level interpretation, not one single implementation object + +### 15.3 Why this section matters + +The point of this section is to prevent readers from confusing “the document presents a coherent design” with “every part of the design is already fully materialized in the current branch”. It helps reviewers and readers distinguish between implementation-grounded facts and system-level abstraction. + +## 16. Failure Modes and Observability + +If the document only explains the normal path, it still feels closer to a talk draft than to a technical design note. An online system also needs a clear view of failure modes and observability. + +### 16.1 Typical failure modes + +- **waiting time grows too much** + - the fixed-step window increases queueing time for new requests +- **beam becomes too large** + - sorting, filtering, and candidate management dominate the cost +- **shared / unshared KV is not organized correctly** + - this can lead to memory waste, repeated loading, or wrong cache selection +- **host participation remains too high** + - this directly weakens the benefit of multi-step execution +- **operator-level gains do not materialize** + - this usually means the execution shape is not stable enough yet + +### 16.2 What should be observed first + +Even without a full monitoring chapter, the document should at least clarify the key observation points: + +- end-to-end request latency +- per-round decode latency +- scheduler overhead ratio +- host-device round trips or synchronization points +- beam-search / cache-select share in total latency +- memory peak and fragmentation tendency + +### 16.3 Shortest debugging path + +When behavior does not match the design expectation, the following order is often the shortest path: + +1. verify that the request actually matches the fixed-step workload assumptions; +2. verify that the decode path really enters the multi-round worker pipeline; +3. verify whether beam search and cache select dominate the cost; +4. finally verify whether xAttention and KV organization actually reduce repeated loading and copying. + +## 17. Benchmark and Acceptance Protocol + +To make this design document more useful as a review and acceptance baseline, it is helpful to include a lightweight benchmark and acceptance template. + +### 17.1 Recommended workload coverage + +- **long input / short output** + - the most representative generative recommendation scenario +- **different beam_width settings** + - for example, one moderate-beam case and one large-beam case +- **different top_k settings** + - useful for observing how search and filtering cost scales + +### 17.2 Recommended metric set + +- throughput +- P50 / P95 / P99 latency +- per-round decode cost +- scheduler overhead ratio +- host participation overhead +- peak memory usage + +### 17.3 Acceptance focus + +- whether fixed-step scheduling really reduces scheduling disturbance +- whether multi-step execution really reduces host-side round control overhead +- whether `xAttention` really improves shared-prefix reuse +- whether beam-related optimization really reduces search and filtering cost + +The value of this section is that it turns “the design sounds reasonable” into “the design can be evaluated with a stable acceptance template”. + +## 18. Key Parameters and Suggested Ranges + +To keep this document from remaining purely conceptual, it is useful to end with a lightweight parameter-oriented section. The purpose is not to declare one universally correct configuration, but to summarize which parameters matter the most and why. + +### 18.1 `beam_width` + +`beam_width` controls how many candidate branches survive after each round. It is one of the most important cost drivers in generative recommendation decode. + +- when `beam_width` is smaller: + - search cost is lower + - candidate coverage is weaker + - lower latency is easier to achieve +- when `beam_width` is larger: + - candidate coverage is stronger + - sorting, filtering, and cache-select cost all increase + - the gain of fixed-step and multi-step execution becomes more visible, but decode-side cost also grows quickly + +So `beam_width` should be treated as a core trade-off parameter among recommendation quality, total latency, and system overhead. + +### 18.2 `top_k` + +`top_k` controls how many candidates are expanded from each beam in one round. +In a large candidate space, increasing `top_k` directly amplifies: + +- sorting work +- invalid-item filtering work +- intermediate candidate-state management cost + +If `top_k` is too large and invalid-item filtering happens too late, then a large amount of computation is spent on candidates that will never survive into the next round. + +### 18.3 Decode-round count + +The more stable the decode-round count is, the easier it is for fixed-step scheduling and multi-step execution to keep producing stable gains. +If the number of rounds stays within a small and predictable range, then: + +- device-side structures can be allocated earlier; +- `step_meta` remains easier to stabilize; +- whole-graph multi-step execution becomes easier to exploit. + +If the round count itself becomes unstable, one of the core assumptions behind the current design becomes weaker. + +### 18.4 Multi-stream concurrency + +More streams do not automatically mean better performance. Too few streams may fail to hide waiting cost, while too many streams may introduce additional contention, scheduling overhead, and resource competition. + +A more reasonable tuning order is usually: + +1. confirm that fixed-step scheduling is already stable; +2. confirm that host-side waiting and queueing are actually significant; +3. only then increase multi-stream or multi-pipeline concurrency. + +### 18.5 Suggested parameter-tuning order + +If this path is tuned further later, the following order is usually more effective: + +1. first confirm that the decode-round count is stable; +2. then tune `beam_width`; +3. then tune `top_k`; +4. only after that tune multi-stream and system overlap. + +The reason is simple: the first two primarily determine workload shape, while the latter ones optimize the system under that shape. + +## 19. References + +This document is implementation-oriented, but some of the background problems, serving ideas, and memory/graph execution context naturally come from public references. If the document is extended later, the following materials are useful anchor points. + +### 19.1 Scheduling and serving systems + +- *Orca: A Distributed Serving System for Transformer-Based Generative Models* + - useful as the background reference for continuous batching + - useful as a comparison point when explaining why REC prefers fixed-step execution + +### 19.2 Memory management and KV cache + +- *Efficient Memory Management for Large Language Model Serving with PagedAttention* + - useful as a background reference for block-based KV management + - useful when explaining why shared/unshared KV separation matters + +### 19.3 Generative recommendation models + +- OneRec + - useful when describing the structure of recall-oriented generative recommendation models +- OneTrans + - useful when describing the structure of ranking-oriented generative recommendation models + +### 19.4 Graph execution and multi-step execution + +- the existing `Graph Mode` design document in this repository + - useful for graph capture / replay, parameterization, and Piecewise Graph +- the current generative recommendation design document + - useful for fixed-step scheduling, multi-round decode, beam search, and xAttention as one coherent serving stack + +### 19.5 Suggested use + +If the document is extended further later: + +- read Orca first when writing scheduling background; +- read PagedAttention first when writing KV / memory layout background; +- return to OneRec / OneTrans when writing model-structure background; +- return to this document and the code-anchor appendix when writing implementation mapping. + +This keeps future expansion grounded and prevents the document from turning into a loose collection of disconnected explanations. + diff --git a/docs/src/content/docs/en/design/graph_mode_design.md b/docs/src/content/docs/en/design/graph_mode_design.md new file mode 100644 index 0000000000..8108a623a3 --- /dev/null +++ b/docs/src/content/docs/en/design/graph_mode_design.md @@ -0,0 +1,478 @@ +--- +title: "Graph Mode Design Document" +sidebar: + order: 1 +--- +## Overview + +xLLM's Graph Mode supports multiple graph execution backends. Its goal is to turn the original Host-driven stream of fine-grained kernel launches into a capture-then-replay execution flow in inference serving, thereby reducing Host scheduling overhead, reducing device-side bubbles, and improving throughput and latency stability. + +This document is intended for developers who need to understand the implementation principles and key design choices. It focuses on: + +- the basic Graph Mode mechanism and how xLLM applies it +- dynamic dimension parameterization +- Piecewise Graph +- a multi-shape reusable memory pool, including input tensor reuse + +This document focuses on the unified Graph Mode design in xLLM and does not expand on backend-specific platform differences. + +The design goals of this document are: + +- provide a unified Graph Mode abstraction across xLLM backends +- explain the three key designs: dynamic dimension parameterization, Piecewise Graph, and multi-shape memory reuse +- clarify what problem each design solves, what assumptions it depends on, and where its boundary lies + +The non-goals of this document are: + +- full adaptation details for every operator or every model +- replacing feature documentation for flags and usage examples + +Related design documents: + +- for a recommendation-oriented case study that focuses on fixed scheduling, multi-step execution, and custom operators, see: [Generative Recommendation Design Document](/en/design/generative_recommendation_design/) + +## 1. Graph Mode Fundamentals + +### 1.1 Capture / Replay Basics + +In traditional eager execution, one forward pass launches many kernels, memory copies, and synchronization operations from the Host. For decode-like workloads, where each step is small but requests are frequent, Host scheduling overhead becomes significant and device-side bubbles become more visible. + +The core idea of Graph Mode is: + +1. **Capture**: when a shape bucket is seen for the first time, run one forward pass on a dedicated stream and record the kernel launches, memory operations, and dependencies into a graph. +2. **Replay**: for later requests that hit the same bucket, replay the recorded graph instead of launching kernels one by one from the Host. + +![Graph Mode](../../assets/graph_mode.png) + +This mechanism usually requires: + +- **Stable execution path** + - graph capture records one concrete execution path; once capture completes, the control flow, launch shape, and dependencies on that path are fixed + - therefore, the capture path cannot switch to a different dynamic branch during replay +- **Stable addresses for key tensors** + - key tensor addresses written into the graph during capture must still be valid at replay time +- **Stable operator behavior and result semantics** + - replay correctness depends on operators along the path being Graph Mode compatible and not changing semantics because of runtime condition changes + +### 1.2 xLLM GraphMode Runtime Foundations + +Starting from the three basic requirements in 1.1, the first two are mainly handled by the xLLM runtime: turning dynamic requests into execution units that can be replayed stably, and ensuring replay still accesses the fixed addresses recorded during capture. For that reason, xLLM introduces a unified Graph Executor in Graph Mode to centralize bucketing, persistent buffers, graph cache management, and the capture / replay lifecycle. + +xLLM's runtime-side foundation work mainly includes: + +- **Graph selection and execution scheduling around execution-path stability** + - requests must first be grouped into buckets by `num_tokens` or a nearby shape + - graph cache is maintained per bucket: replay directly on a hit, otherwise capture first and cache the graph + - use full graphs for paths that are fully capturable, and switch to Piecewise Graph when part of the path breaks graph capture +- **Persistent buffer and graph-instance management around address stability** + - dynamic inputs such as tokens, positions, seq_lens, and block_tables cannot be reallocated to arbitrary new addresses before replay; xLLM must write them into persistent buffers first and then update content at fixed addresses + - tensors allocated temporarily during model graph construction also need to be retained with the corresponding graph instance instead of being reclaimed when their local scope ends + - when the shared memory pool is enabled, captures of different shapes can reuse the same underlying physical memory, while the virtual addresses seen by the graph still remain stable +- **A unified Graph Executor abstraction** + - capture, cache, replay, graph-instance management, and backend abstraction all need to be handled centrally by the runtime + - the point of this layer is to pull Graph Mode runtime orchestration out of model code rather than scattering it across layers or operators + +In concrete execution, the Graph Executor runtime flow can be summarized as: + +1. **Bucket the request**: group the request into a bucket by `num_tokens` or a nearby shape. +2. **Prepare inputs**: write tokens, positions, seq_lens, and block tables into persistent buffers, and update runtime metadata such as `attn_metadata` and `plan_info`. +3. **Choose capture or replay**: replay directly if the bucket already has a cached graph; otherwise capture, cache, and then enter the replay path. +4. **Execute the graph**: run a full graph or a piecewise graph depending on the scenario. + +The third requirement in 1.1, "stable operator behavior and result semantics", falls more on model-side and operator-side adaptation, which is covered in 1.3. + +### 1.3 Model Adaptation Required for Graph Mode + +After the xLLM runtime takes care of bucketing, buffers, and graph-instance management, the model side still needs to satisfy the third requirement from 1.1: operator behavior and result semantics must remain stable across capture / replay and must not break because of host participation, dispatch changes, or launch-shape changes. + +The required model adaptations mainly include: + +- **Remove operations that are not capture-safe** + - General requirement: the capture path must not contain operations that trigger host-side decisions, implicit host-device synchronization, or extra control-flow changes, such as stream synchronization, logic branches driven by host tensors, or implicit host-device interactions + - Typical example 1: operators may depend on host tensor contents to decide task partitioning or execution mode. For example, ATB [PA](https://www.hiascend.com/document/detail/zh/canncommercial/83RC1/API/ascendtbapi/ascendtb_01_0197.html) and [MLA](https://www.hiascend.com/document/detail/zh/canncommercial/83RC1/API/ascendtbapi/ascendtb_01_0314.html) in Ascend CANN commercial 8.3 can choose different task layouts based on host-side `kv_seq_lens` and `q_seq_lens` + - Typical example 2: a host scalar participates in tensor computation, or the host directly reads tensor data, for example: + +```cpp +Tensor a; +Tensor b = a * 0.5; // the scalar is implicitly transferred to the tensor's device, causing a synchronized H2D + +torch::Tensor max_of_seq = torch::max(input_params.kv_seq_lens); +max_seq_len_ = std::max(max_of_seq.item(), max_seq_len_); // host reads tensor data +``` + +- **Adapt operators** + - General requirement: the kernel execution path must remain stable for the same input shape. For one input shape, graph execution should choose the same kernel every time, without mixing prefill, chunked prefill, and decode behavior. At the same time, `grid_dim`, `block_dim`, task count, workspace shape, and tiling content must also remain stable + - Typical example: in Ascend CANN commercial 8.3, ATB PA may decide at runtime whether to enable flash-decoding long-sequence mode based on sequence length and batch size. The long-sequence and short-sequence modes use different tiling keys, and the actual dispatched `kernel_name` is also different. This kind of behavior, where the same business path triggers different kernels under different runtime conditions, must be converged before entering Graph Mode + +## 2. Dynamic Dimension Parameterization + +### 2.1 Problem + +Graph capture records a fixed kernel sequence and the launch parameters of every kernel. At replay time, the runtime can only repeat that recorded launch shape. It cannot re-decide `grid_dim`, `block_dim`, task count, or the tiling path inside the same graph. + +As a result, not every dynamic request dimension can continue to vary once the request enters Graph Mode. For attention, the real dynamic factors usually include more than `num_tokens`: they also include `batch_size`, `q_seq_lens`, `kv_seq_lens`, and `block_tables_size`. If these dimensions further affect task partitioning, workspace layout, `tiling_params`, or kernel path, replay may reuse the stale capture-time configuration and become incorrect. + +The core question of this chapter is therefore: which dynamic information can still be updated after the graph has been selected, and which dimensions must instead be handled by finer-grained bucketing or by building another graph. + +### 2.2 Solution + +The goal of dynamic dimension parameterization is not to make one graph accept arbitrary `num_tokens`. Instead, once the graph has already been selected by `bucket_num_tokens`, the goal is to let the same bucket safely cover more real dynamic requests. + +#### 2.2.1 Parameterization Boundary + +Whether a dimension can still vary at replay time does not depend on whether it is dynamic in the request. It depends on whether that dimension has already been folded into the execution shape during capture. + +Dynamic factors can be divided into two categories: + +1. **Dimensions that determine graph shape** + - once a dimension enters `grid_dim`, `block_dim`, task count, workspace layout, tiling key, or `tiling_params`, it becomes part of the capture result + - after that, it usually cannot be changed inside the same graph + +2. **Dimensions that can still be updated after graph selection** + - such as `batch_size`, `q_seq_lens`, `kv_seq_lens`, `block_tables`, `new_cache_slots`, and `plan_info` + - they are suitable parameterization targets only if they do not further rewrite execution shape + +A simplified norm-kernel example is: + +```cpp +int grid_dim = num_tokens; +NormKernel<<>>(x, y, ...); +``` + +Here, `grid_dim` and `block_dim` are both launch parameters. Once `num_tokens` participates in launch configuration, it becomes part of graph shape; after capture, replay can no longer change `grid_dim = 128` into `grid_dim = 256` within the same graph. + +So the boundary is straightforward: any dynamic factor that changes launch shape, workspace, or tiling path should not be handled only by updating parameters before replay. + +#### 2.2.2 How xLLM Handles It + +xLLM handles this in two layers. + +The first layer handles `num_tokens`: xLLM uses bucketing and padding to normalize the raw request into a fixed `bucket_num_tokens`, and then selects the graph accordingly. + +The second layer handles the remaining dynamic information inside that bucket: adapt operators that depend on host tensors, host planning, or host-side tiling so that request-varying information is stored in device-side persistent buffers, or otherwise externalized into data that can be refreshed before replay, instead of letting replay stay bound to the old host-side planning result from capture. + +Take NPU attention as an example. A common pre-adaptation problem is that `q_seq_lens` and `kv_seq_lens` first participate in host-side planning, which produces task, workspace, or tiling-related data before launching the kernel: + +```cpp +// Host side +auto plan_info = PlanAttention(q_seq_lens_host, kv_seq_lens_host, ...); +WritePlanToWorkspace(attention_workspace, plan_info); + +AttentionKernel<<>>( + q, k, v, out, + attention_workspace, + tiling_params); +``` + +In this kind of implementation, graph replay does not automatically redo planning, so it can only repeat the launch shape and workspace layout from capture time. The adapted direction is to write request-varying information such as `q_seq_lens`, `kv_seq_lens`, `block_tables_size`, and `plan_info` into device-side persistent buffers under a fixed launch shape, and let the kernel read them directly. For example, a parameterized `kv_seq_lens` path may look like: + +```cpp +struct AttentionLaunchArgs { + void* q; + void* k; + void* v; + void* out; + int32_t batch_size; + int32_t* kv_seq_lens; // device buffer + void* attention_workspace; + void* tiling_params; +}; + +LaunchAttentionKernel(args); +``` + +If a dynamic factor changes task count, array length, workspace shape, or tiling path as soon as it varies, xLLM does not force it into "parameterization updates". Instead, it uses one of two approaches: + +1. expand the bucketing dimensions and include those factors in the graph-selection key, for example a combination of `num_tokens` and `num_comm_tokens` +2. keep that dynamic-shape-sensitive part outside the full graph and handle it with Piecewise Graph + +### 2.3 Result + +Dynamic dimension parameterization directly brings the following result: one `num_tokens` bucket can safely cover more real requests, and under a stable execution shape, operators such as attention can still see the correct dynamic information at replay time, including `batch_size`, `seq_lens`, `block_tables_size`, and `plan_info`. + +One thing to keep in mind is that parameterization does not eliminate bucketing. The dynamic nature of `num_tokens` is still mainly absorbed by graph selection; if a bucket still contains other factors that rewrite tiling keys, task count, communication scale, or kernel path, those factors must still enter the bucketing key or move to Piecewise Graph. Communication-heavy scenarios are a typical example. In Attention DP + MoE EP, communication scale may depend on the maximum DP data size rather than strictly matching local `num_tokens` on one device. + +## 3. Piecewise Graph + +### 3.1 Problem + +In prefill and chunked prefill scenarios, the full forward path is usually more complex than decode. Some operators may: + +- fail to be captured stably +- be highly sensitive to dynamic metadata +- rely on extra runtime preparation during capture + +If the requirement is that the entire path must be capturable as one full graph, Graph Mode coverage becomes very limited. Once one critical operator breaks graph capture, the whole path loses graph execution. + +### 3.2 Solution + +#### 3.2.1 Underlying Logic + +One common question in implementation is: why can decode use a full graph while chunked prefill usually needs Piecewise Graph? + +The key difference is attention. + +Under current decode semantics, one step usually means one generated token per sequence, so `num_tokens` and `batch_size` are almost one-to-one. For attention, that means: + +- when `num_tokens` grows, `batch_size` grows with it +- the pattern of `q_seq_lens` stays relatively stable +- task partitioning, workspace requirements, and launch arguments can mostly be bucketed together with `num_tokens` + +Therefore, when building buckets by `num_tokens`, decode satisfies the reuse conditions for a full graph. + +In chunked prefill, however, `num_tokens` and `batch_size` are no longer tied together. The same `num_tokens` may correspond to very different request combinations, for example: + +- `num_tokens = 128, batch_size = 2, q_seq_lens = [64, 64]` +- `num_tokens = 128, batch_size = 4, q_seq_lens = [32, 32, 32, 32]` +- `num_tokens = 128, batch_size = 8, q_seq_lens = [16, 16, 16, 16, 16, 16, 16, 16]` + +For attention, these requests share the same `num_tokens` but they are not the same shape. Attention task partitioning, local indexing, the lengths of `q_seq_lens` and `kv_seq_lens`, and the corresponding `plan_info` all directly depend on `batch_size`. If the system reuses a previously captured graph only by `num_tokens`, then replay may keep using the old batch partitioning and old task partitioning from capture, which can make the result incorrect. + +`batch_size` is hard to parameterize here because it changes more than just one scalar value. It changes the structure of the launch arguments themselves: + +- the lengths of `q_seq_lens` and `kv_seq_lens` change +- the task count and task boundaries of attention change +- `plan_info` and workspace layout change +- the indexing and partition logic inside the kernel change + +In other words, in chunked prefill, `batch_size` is not a dimension that can be handled by updating a few parameters. It changes the execution shape of the attention part itself. Unless `num_tokens × batch_size`, or even more fine-grained `seq_lens` combinations, are added into bucketing, selecting graphs only by `num_tokens` is not safe. + +That is why the current design is better suited to: + +- **decode**: attention can be captured together with the full graph +- **chunked prefill**: attention stays outside the graph while the more stable surrounding parts enter Piecewise Graph + +#### 3.2.2 Core Idea + +The goal of Piecewise Graph is not to force full-graph capture. It is to split one full execution path into several pieces: + +- graph-friendly parts: captured into subgraphs +- parts that are not suitable for graph capture: kept in eager mode + +At replay time, those pieces are executed in the same order recorded during capture. + +This preserves as much Graph Mode benefit as possible even when full-graph capture is not feasible. + +The following diagram uses a three-layer Qwen3 decoder as an example. Attention stays as an independent runner, while the non-attention operators between attention calls are packed as tightly as possible into continuous graph pieces. + +![Piecewise Graph Diagram](../../assets/piecewise_graph_diagram.svg) + +#### 3.2.3 How xLLM Implements It + +In xLLM, Piecewise Graph is mainly used for prefill-like scenarios and is controlled by `enable_prefill_piecewise_graph`. + +Its core implementation is: + +1. use `PiecewiseGraphs` to maintain the replay instruction sequence +2. store capturable segments as graph objects +3. record non-graph attention segments as runners through `AttentionRunner` +4. replay in the original order as `graph -> runner -> graph -> ...` + +The key point is how attention is handled: + +- during piecewise capture, attention temporarily ends the current graph capture +- attention itself is not executed during capture; instead, the tensors, workspace, and parameters required for attention are recorded +- at replay time, attention runs with the latest `plan_info`, `q_cu_seq_lens`, and `kv_cu_seq_lens` + +The direct reason is that, in chunked prefill, attention has dynamic dimensions beyond `num_tokens`, while current graph bucketing is mainly organized by `num_tokens`. Keeping attention outside the graph avoids replay errors where the graph hits by `num_tokens`, but `batch_size` and `seq_lens` have already changed. + +So the essence of Piecewise Graph is not to support all dynamic attention logic inside the graph. It is to: + +- put the stably capturable parts into graphs +- keep the more dynamic-shape-sensitive attention as a separate execution unit +- preserve the original execution order through one unified instruction sequence + +### 3.3 Result + +The results and boundaries of this design are reflected in the following aspects. + +#### 3.3.1 Applicable Scenarios + +- prefill +- chunked prefill +- scenarios with local graph breaks where graph execution benefit is still desired for the overall path + +#### 3.3.2 Benefits + +- more flexible than all-or-nothing full-graph capture +- significantly expands Graph Mode coverage +- fits model structures where attention is highly dynamic while parts such as MLP remain relatively stable + +#### 3.3.3 Limitations + +- currently focused mainly on prefill-like paths +- replay depends on correctly updated `attn_metadata.plan_info` + +## 4. Multi-Shape Reusable Memory Pool + +### 4.1 Problem + +In multi-shape scenarios, Graph Mode faces two problems at the same time: + +1. **capture-time memory grows linearly with the number of shapes** + - each new shape entering capture usually gets its own independent memory buffer pool + - because Graph Mode requires stable addresses between capture and replay, these buffers often cannot be released in time + - memory usage grows from the expected `max(shape)` to `sum(shape)` + +2. **input-side memory is also repeatedly duplicated** + - if each shape keeps its own tokens, positions, seq_lens, block_tables, and related tensors + - then even if the graph itself can be reused, input-side memory still accumulates by shape + +So the reusable memory pool is not just about saving memory. It is about supporting dynamic new shapes while making memory usage converge from `sum(shape)` to `max(shape)`. + +The concrete goals include: + +1. **capture-time memory reuse**: different graphs share the same underlying physical memory +2. **input-side memory reuse**: replays of different shapes share one set of persistent input buffers +3. **no address conflicts**: every captured graph still keeps its own stable virtual address view +4. **on-demand support for new shapes**: a new shape can still be captured without invalidating existing graphs +5. **no extra replay overhead**: most reuse happens during capture and input preparation, while replay stays as unchanged as possible + +#### 4.1.1 Root Cause + +- on the CUDA path, the allocator typically allocates an independent memory buffer pool for every graph capture +- Graph Mode then requires stable addresses between capture and replay, so buffers from old shapes cannot be reclaimed like ordinary eager temporary memory +- as more shapes enter capture, memory usage grows from the expected `max(shape)` to `sum(shape)` + +#### 4.1.2 Constraints + +- under the current assumption, graphs of different shapes do not replay simultaneously +- dynamically appearing new shapes must still be supported through on-demand capture +- existing graph caches should not be torn down just because a new shape appears +- replay should not become more complex because of memory reuse + +#### 4.1.3 Failed Approach and Technical Challenge + +A more direct idea is to reset the allocation pointer before each capture so that different shapes reuse the same virtual address space. + +But this is not safe in practice. Allocators usually track allocated blocks by address. If a new capture reuses an old address, it may overwrite the previous address record. Then, when tensors belonging to an old graph are later destructed or freed, the address record may already be gone, which can trigger errors such as `invalid device pointer`. + +This reveals the real challenge: what Graph Mode truly needs to keep stable is the address view seen by each graph, while what really needs to be reused is the underlying physical memory. So the problem is not just memory saving. It is how to satisfy all of the following at the same time: + +- the virtual address spaces of captured graphs do not conflict +- the underlying physical memory can still be shared across shapes +- adding a new shape into capture does not break replayability of already captured graphs + +### 4.2 Solution + +The core idea is not to let different shapes share one virtual address range. Instead, it is to let them share one set of physical memory while each captured graph keeps its own virtual address view. + +The reason is simple: if different shapes are forced to reuse the same virtual address space, allocator address tracking will conflict, and old graphs may no longer find their original address records when they are released or destroyed. In other words, the thing that must be reused is the underlying physical memory, not the virtual address seen by the graph itself. + +To show why xLLM chooses "shared physical memory + independent virtual address spaces", the common solutions can be compared as follows: + +| Solution | Memory reuse mechanism | Physical memory outcome | Address-conflict risk / mitigation | Main limitation | +|----------|------------------------|-------------------------|------------------------------------|-----------------| +| vLLM | Shared graph memory pool + capture larger shapes first | may still approach `sum(shape)` | relies on a shared memory pool and free-block management, usually without explicit address conflicts | reuse efficiency depends on capture order and allocator behavior | +| SGLang | Global graph memory pool + capture larger shapes first | may still approach `sum(shape)` | relies on global pool or graph-private pool management to avoid conflicts | memory still grows when there are many distinct shapes | +| xLLM | Map multiple virtual address spaces to one set of physical memory | converges to `max(shape)` | every graph keeps an independent address view, avoiding address-record conflicts at the root | depends on VMM and allocator integration support from the runtime | + +xLLM does not reuse the same virtual address space. Instead, each graph keeps its own address view, while the truly shareable part is pushed down into underlying physical memory. + +#### 4.2.1 Multi-Shape Input Tensor Reuse + +xLLM first solves repeated input-side memory usage. Instead of allocating independent input tensors for each shape, it pre-allocates one set of persistent buffers for the maximum shape and lets different shapes share them. + +These buffers usually include: + +- `tokens` +- `positions` +- `q_seq_lens` +- `kv_seq_lens` +- `block_tables` +- `new_cache_slots` +- `hidden_states` + +Replay-time handling is straightforward: + +1. write the actual input of the current request into the prefix of the buffer +2. zero-fill padding regions when needed +3. construct slice views for the actual shape + +As a result, input-side memory no longer accumulates by shape and stays at one `max(shape)` allocation. + +#### 4.2.2 Capture-Time Memory Reuse + +After solving input duplication, xLLM still needs to solve the memory accumulation caused by graph capture itself. + +The core design is: + +- every capture uses a **new virtual address space** +- different virtual address spaces map to the **same physical memory** + +This satisfies two requirements at once: + +1. **stable addresses from the graph's point of view**: each captured graph sees its own fixed virtual addresses +2. **only one physical-memory footprint**: the underlying physical memory grows by the maximum demand instead of accumulating by shape + +Compared with reusing one virtual address range, this design avoids address-tracking conflicts and therefore works better with existing allocator mechanisms. + +This design relies on a unified virtual memory management capability, including: + +- reserving a new virtual address space +- creating or expanding underlying physical memory +- mapping the same physical memory into multiple virtual address spaces +- performing unmap, address-space release, and access-permission operations when needed + +The key property is that the same physical memory can be mapped into multiple virtual address spaces. That is exactly what makes "different graphs have independent addresses while still sharing the same physical memory" possible. + +#### 4.2.2.1 Address Mapping Relation + +The core mapping relation for capture-time memory reuse is shown below: + +![Shared virtual-to-physical mapping](../../assets/shared_vmm_mapping_diagram.svg) + +This diagram shows that, as the shape grows, each new capture switches to a new virtual address space, while all three virtual spaces reuse the same physical memory pool. Different shapes only use different portions of that pool, and the total physical memory converges to `max(shape)`. + +### 4.3 Results + +#### 4.3.1 Memory Efficiency Metrics + +| Metric | Without memory reuse | With memory reuse | +|--------|----------------------|-------------------| +| Physical memory size | `sum(shape)` | `max(shape)` ✅ | +| Virtual memory | multiple copies | multiple copies | +| Stability | address-conflict risk | stable ✅ | + +The key change is that physical memory converges to `max(shape)`, while virtual address space still grows with the number of captured shapes. + +#### 4.3.2 Performance Impact + +- **capture stage**: switching virtual address spaces and extending mappings introduces extra overhead +- **replay stage**: no extra graph execution overhead, and behavior is essentially the same as normal Graph Mode replay +- **input preparation stage**: one extra write into persistent buffers plus slice-view construction, which is still typically cheaper than repeated allocation + +#### 4.3.3 Virtual Address Space Overhead + +One cost of this design is that virtual address space grows as more shapes are captured. But this growth is address-space usage, not physical memory usage. + +On 64-bit systems, virtual address space is usually much larger than single-device memory capacity, so it is typically not the primary bottleneck for this kind of graph capture scenario. In other words, this design uses abundant address space to bring physical memory down from `sum(shape)` to `max(shape)`. + +#### 4.3.4 Result Validation + +- captures of different shapes can share the same underlying physical memory +- every capture still keeps an independent virtual address space instead of reusing the same address view +- address-conflict issues are no longer exposed through the failed "reuse one virtual address space" path +- memory usage converges from `sum(shape)` to `max(shape)` + +#### 4.3.5 Practical Effects + +- in multi-shape scenarios, memory behavior changes from "grows linearly with captures" to "physical memory converges to the maximum shape" +- graph reuse becomes more stable because direct reuse of one virtual address space is avoided +- the design supports on-demand capture for new shapes without tearing down existing graph caches +- replay stays unchanged or nearly unchanged, while most added complexity is concentrated in capture and input preparation + +#### 4.3.6 Assumptions and Boundaries + +- virtual address space still grows with the number of captured shapes, even though physical memory only grows with the maximum demand +- the design depends on runtime support for virtual address management, physical memory mapping, and allocator integration +- it currently assumes that graphs of different shapes do not replay simultaneously; that assumption is what makes physical memory sharing safe +- the current implementation still keeps physical memory and virtual spaces for the full lifecycle instead of doing fine-grained dynamic release + +## 5. References + +- [ACLGraph Capture / Replay mechanism](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/83RC1alpha002/appdevg/acldevg/aclcppdevg_000519.html) +- [CudaGraph capture mechanism](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#cuda-graphs) +- [vLLM graph execution reference](https://github.com/vllm-project/vllm/blob/main/vllm/v1/worker/gpu/cudagraph_utils.py) +- [SGLang graph execution reference](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/model_executor/cuda_graph_runner.py) + diff --git a/docs/src/content/docs/en/dev_guide/ai_coding_workflow.md b/docs/src/content/docs/en/dev_guide/ai_coding_workflow.md new file mode 100644 index 0000000000..87e6d80f84 --- /dev/null +++ b/docs/src/content/docs/en/dev_guide/ai_coding_workflow.md @@ -0,0 +1,110 @@ +--- +title: "AI Coding Workflow" +sidebar: + order: 4 +--- + +This guide summarizes a practical workflow for xLLM developers who need to +optimize NPU serving, debug regressions, or prepare pull requests with +reproducible evidence. + +The workflow is maintained as a separate knowledge base at +[:simple-github: xllm-workflow](https://github.com/xLLM-AI/xllm-workflow). Use +that repository when you want copy-ready agent skills, prompt templates, +artifact schemas, and reusable model optimization history. + +![xLLM AI Coding Workflow](../../assets/xllm-ai-coding-workflow-en.png) + +## When to use it + +Use this workflow when a change needs evidence beyond a normal code review: + +- optimizing TTFT, TPOT, TPS, memory usage, or serving concurrency; +- comparing xLLM with vLLM-Ascend, SGLang NPU, or another serving framework; +- debugging garbled output, dataset score drops, GPU/NPU mismatches, OOM, + graph replay failures, HCCL issues, or runtime crashes; +- validating an NPU-related PR before it is merged; +- deciding whether an operator migration or kernel-level experiment is needed. + +Do not treat one smoke run as a formal conclusion. Formal performance and +accuracy claims should include the exact command, environment, workload, raw +artifacts, and normalized summaries. + +## Evidence loop + +For performance and correctness-sensitive work, follow this loop: + +```text +target -> baseline -> profiling -> patch -> accuracy -> performance -> record +``` + +The key rule is to keep benchmark, profiling, and accuracy evidence separate. +Profiling explains bottlenecks, but it does not replace warmed-up before/after +performance measurements. + +Use these phases for a complete optimization task: + +| Phase | Purpose | Output | +|---|---|---| +| Target and environment | Define the goal, model, framework commit, hardware, CANN/runtime versions, workload, and SLA. | Run manifest | +| Historical knowledge | Check prior model PRs, failed attempts, and known risky paths. | History notes | +| Fair baseline | Run warmed-up baseline tests before code or parameter changes. | Raw metrics and summary | +| Evidence collection | Collect profiling, capacity, pipeline, compute, or accuracy evidence based on the symptom. | Diagnostic report | +| Patch | Make one meaningful, reviewable change per round whenever practical. | Code diff | +| Validate | Re-run accuracy, performance, build, and UT checks appropriate to the change. | Validation table | +| Record | Preserve commands, metrics, failed attempts, risk notes, and follow-up work. | Reusable lesson | + +## Developer checklist + +Before opening or updating an NPU optimization PR, make sure the PR description +can answer these questions: + +- What model, tokenizer, dtype, device type, device count, and framework commit + were used? +- What exact startup command and benchmark command were used? +- Was the baseline warmed up and run on clean devices? +- Are profiling results used only as diagnostic evidence? +- Which accuracy level was run, and where are failed cases stored if any? +- What changed in the patch, and what risk remains? +- Which artifacts can another developer use to replay the conclusion? + +## Recommended artifacts + +For formal results, keep these artifacts together under the same run root: + +- `manifest.md` or `manifest.yaml` with environment and command details; +- raw evalscope or benchmark output; +- normalized `metrics.json` or `summary.md`; +- profiling report and timeline notes when profiling is used; +- `failed_cases.jsonl` or equivalent bad-case records for accuracy work; +- PR notes that summarize what changed, why it is safe, and how it was + validated. + +The workflow repository provides shared schemas for these artifacts: + +- `references/run-manifest-template.md` +- `references/perf-artifact-schema.md` +- `references/profiling-artifact-schema.md` +- `references/accuracy-artifact-schema.md` + +## Related skills + +The workflow repository includes task-oriented skills that can be loaded by +Codex, Claude Code, opencode, or another local agent runtime: + +| Task | Skill | +|---|---| +| End-to-end optimization | `xllm-npu-sota-loop` | +| Service startup and evalscope collection | `xllm-npu-eval-runner` | +| Fair framework comparison | `xllm-npu-benchmark` | +| msprof / MindStudio analysis | `xllm-npu-profiler` | +| Decode bubble and rank-skew analysis | `xllm-npu-pipeline-analysis` | +| HBM, KV cache, and OOM analysis | `xllm-npu-capacity-planner` | +| FLOPs, MFU, and lower-bound estimates | `xllm-npu-compute-simulation` | +| Accuracy regression debugging | `xllm-npu-accuracy-debug` | +| Crash or runtime incident triage | `xllm-npu-incident-triage` | +| NPU code review | `xllm-npu-code-review` | +| Operator migration | `xllm-npu-op-migration` | + +These skills are aids for disciplined engineering. The final judgment still +depends on reproducible xLLM artifacts and reviewable code changes. diff --git a/docs/src/content/docs/en/dev_guide/code_arch.md b/docs/src/content/docs/en/dev_guide/code_arch.md new file mode 100644 index 0000000000..cb34c132ff --- /dev/null +++ b/docs/src/content/docs/en/dev_guide/code_arch.md @@ -0,0 +1,31 @@ +--- +title: "Code Architecture" +sidebar: + order: 1 +--- +``` +├── xllm/ +| : main source folder +│ ├── api_service/ # code for api services +│ ├── core/ +│ │ : xllm core features folder +│ │ ├── common/ +│ │ ├── distributed_runtime/ # code for distributed and pd serving +│ │ ├── framework/ # code for execution orchestration +│ │ ├── kernels/ # adaption for npu kernels adaption +│ │ ├── layers/ # model layers impl +│ │ ├── platform/ # adaption for various platform +│ │ ├── runtime/ # code for worker and executor +│ │ ├── scheduler/ # code for batch and pd scheduler +│ │ └── util/ +│ ├── function_call # code for tool call parser +│ ├── models/ # models impl +│ ├── processors/ # code for vlm pre-processing +│ ├── proto/ # communication protocol +│ ├── pybind/ # code for python bind +| └── server/ # xLLM server +├── examples/ # examples of calling xLLM +├── tools/ # code for npu time generations +└── xllm.cpp # entrypoint of xLLM +``` + diff --git a/docs/src/content/docs/en/dev_guide/online_profiling.md b/docs/src/content/docs/en/dev_guide/online_profiling.md new file mode 100644 index 0000000000..29d44ac6f8 --- /dev/null +++ b/docs/src/content/docs/en/dev_guide/online_profiling.md @@ -0,0 +1,106 @@ +--- +title: "Online Profiling" +sidebar: + order: 3 +--- + +## Background + +Timeline profiling is essential for diagnosing performance bottlenecks in an online serving deployment: where time is spent across host scheduling, device kernels, and communication. vLLM exposes this through two HTTP endpoints, `POST /start_profile` and `POST /stop_profile`, which toggle profiling on every worker so traces can be collected on a live server without restarting it. + +xLLM provides the equivalent capability with two backends, selected by `--profile_backend`: + +- **`torch` (default)** — records CPU and CUDA activities in-process via libtorch's Kineto profiler (the C++ equivalent of `torch.profiler.profile`) and writes a Chrome trace to disk on `/stop_profile`. No external profiler is required: just launch the server normally and drive the two endpoints. This mirrors vLLM's default `TorchProfilerWrapper`. +- **`cuda`** — only toggles the CUDA profiler capture range (`cudaProfilerStart()` / `cudaProfilerStop()`). It records nothing on its own and must be paired with NVIDIA Nsight Systems (`nsys`): the server is launched under `nsys profile` with a capture range tied to the CUDA Profiler API, and the two endpoints open and close the window that `nsys` records. + +Both backends are CUDA only for now. Support for other backends will be added later. + +## Introduction + +The control flow mirrors the existing `sleep`/`wakeup` broadcast path: + +``` +HTTP POST /start_profile + -> APIService::StartProfileHttp (xllm/api_service) + -> Master::start_profile (xllm/core/distributed_runtime) + -> Engine::start_profile (broadcast to all workers) + -> WorkerClient::start_profile_async (local worker, in-process) + -> RemoteWorker::start_profile_async (remote worker, over brpc) + -> WorkerService::StartProfile (worker server handler) + -> WorkerImpl::start_profile + -> TorchProfiler::start (Kineto, default) + or CudaProfiler::start (cudaProfilerStart, --profile_backend=cuda) +``` + +The engine fans the request out to every worker concurrently and waits for all of them to acknowledge. Because xLLM runs one worker thread per device inside a single process and Kineto/CUPTI is process-global, each profiler is wrapped in a process-wide, idempotent singleton: the first `start` opens the collection window for the whole process, and repeated/overlapping `start`/`stop` calls are coalesced. `/stop_profile` closes the window. + +For the `torch` backend, the profiler is enabled and disabled on the worker's compute thread (the one that runs the forward pass), so host-side CPU operators are captured. On `/stop_profile`, libtorch writes the Chrome trace itself; the file goes to `--profile_dir` (the current working directory when unset). For the `cuda` backend, the trace output location is controlled by `nsys` (its `-o` flag), not by xLLM. + +## Usage + +Profiling is opt-in. Start the server with profiling enabled: + +```shell +--enable_online_profile=true +``` + +`enable_online_profile`, `profile_backend`, and `profile_dir` can also be set via the JSON config file. The endpoints only act when `--enable_online_profile=true`; otherwise they respond with an error explaining how to enable the feature. + +### Default backend (`torch`, no nsys required) + +Just start the server normally and choose where traces are written: + +```shell + \ + --enable_online_profile=true \ + --profile_dir=/path/to/traces # optional; defaults to the current directory +``` + +Then, against the running server: + +```shell +# Start collecting +curl -X POST http://127.0.0.1:9977/start_profile + +# ... send the inference requests you want to profile ... + +# Stop collecting (writes the trace at this point) +curl -X POST http://127.0.0.1:9977/stop_profile +``` + +(Replace the host/port with your server's address.) On `/stop_profile`, each worker writes a Chrome trace named `xllm_rank__.pt.trace.json` into `--profile_dir`. The absolute path is printed in the server log. + +### `cuda` backend (capture-range only, requires nsys) + +To use the capture-range path instead, set `--profile_backend=cuda` and launch the server under `nsys` with a capture range tied to the CUDA Profiler API: + +```shell +# --capture-range=cudaProfilerApi makes nsys start/stop capturing exactly when +# cudaProfilerStart/Stop are called, and --capture-range-end=repeat allows +# multiple start/stop cycles in one session. +nsys profile \ + --trace=cuda,nvtx,osrt \ + --capture-range=cudaProfilerApi \ + --capture-range-end=repeat \ + -o xllm_profile \ + --enable_online_profile=true --profile_backend=cuda +``` + +Drive the window with the same `/start_profile` and `/stop_profile` endpoints. When the server process exits, `nsys` writes the report to `xllm_profile.nsys-rep`. + +## Viewing traces + +For the `torch` backend, open the generated `.pt.trace.json` in [Perfetto](https://ui.perfetto.dev), `chrome://tracing`, or TensorBoard. + +For the `cuda` backend, open the generated `.nsys-rep` in the Nsight Systems GUI, or summarize it on the command line: + +```shell +nsys stats xllm_profile.nsys-rep +``` + +## Notice + +- Profiling is currently supported on **CUDA only**. On other backends the endpoints return an error. +- The two endpoints are only active when `--enable_online_profile=true`; otherwise they respond with an error explaining how to enable the feature. +- With `--profile_backend=cuda`, `cudaProfilerStart`/`cudaProfilerStop` only have an effect when the server is running under a profiler such as `nsys` (or `ncu`) configured with `--capture-range=cudaProfilerApi`. Calling the endpoints without such a profiler attached is harmless but produces no trace. For multi-process / multi-GPU runs, `nsys` recommends launching with `--trace-fork-before-exec=true` so child worker processes are traced. +- Profiling adds runtime overhead. Enable it only for diagnosis, not in steady-state production serving, and keep the capture window short. diff --git a/docs/src/content/docs/en/dev_guide/tilelang_ascend_kernel_dev.md b/docs/src/content/docs/en/dev_guide/tilelang_ascend_kernel_dev.md new file mode 100644 index 0000000000..4a07982dcf --- /dev/null +++ b/docs/src/content/docs/en/dev_guide/tilelang_ascend_kernel_dev.md @@ -0,0 +1,399 @@ +--- +title: "xLLM Ascend TileLang Kernel Development Guide" +sidebar: + order: 2 +--- +This document explains how to add or modify an Ascend TileLang kernel in xLLM. The examples use the current `rope` kernel throughout. + +Relevant directories: + +- Python kernel definitions: `xllm/xllm/compiler/tilelang/targets/ascend/kernels` +- NPU runtime wrappers: `xllm/xllm/core/kernels/npu/tilelang` + +Builds and tests should be run inside the NPU container. + +## 1. First Decide What Kind of Change You Are Making + +- Add a `specialization` + - Add one more compiled parameter combination to an existing kernel + - Reuse the same wrapper, the same runtime dispatch fields, and the same C ABI + - Typical changes are updates to `DISPATCH_SCHEMA` or `SPECIALIZATIONS` +- Add a `kernel` + - Add a new logical operator + - Typical changes are a new Python kernel file, a new wrapper C++ file, and one CMake registration + +For `rope`: + +- Adding one more item like `{"variant_key": "...", "head_dim": ..., "rope_dim": ..., "dtype": ...}` to `SPECIALIZATIONS` means adding a new `specialization` +- Adding a new external interface such as `xxx_wrapper.cpp` means adding a new `kernel` + +## 2. Development Order + +The recommended order is: + +1. Implement the TileLang kernel in a Python file such as `rope.py` +2. Implement `generate_source(...)` to lower the kernel into Ascend-C source +3. Declare `DISPATCH_SCHEMA` and `SPECIALIZATIONS` +4. Generate `registry.inc` once and inspect it +5. Then write or update the runtime specialization construction logic in the wrapper +6. Wire it into CMake and run tests + +The key idea behind this order is: + +- implement the kernel itself first +- then fix the runtime dispatch schema +- then write the wrapper against the generated `registry.inc` + +## 3. Write the Python Kernel + +Using `rope.py` as the example, the Python side can be understood in three layers: + +- `build_rope_kernel(...)`: kernel implementation +- `generate_source(...)`: AOT export +- `RopeKernel`: kernel registration plus dispatch schema and compiled instance declaration + +### 3.1 Implement `build_rope_kernel(...)` + +`build_rope_kernel(...)` is the actual TileLang kernel implementation. This is where you write: + +- `@T.prim_func` +- input and output tensor shapes +- parallel task organization under `with T.Kernel(...)` +- UB allocation and the actual compute logic + +The simplified structure in `rope.py` looks like this: + +```python +def build_rope_kernel( + head_dim: int, + rope_dim: int, + vec_core_num: int, + ub_buffer_bytes: int, +): + task_num = vec_core_num + m_num = vec_core_num // 2 + + @T.prim_func + def rope_in_place_kernel(...): + with T.Kernel(m_num, is_npu=True) as (cid, vid): + task_id = cid * 2 + vid + ... + + return rope_in_place_kernel +``` + +Here, `head_dim` and `rope_dim` are the compile-time parameters that this implementation actually depends on. + +Vector kernels such as `rope` must also follow the fixed-task convention used by the current AOT path. In the current AOT flow, the kernel launch `block_num` is fixed at compile time, which means: + +- runtime input shapes do not change the kernel launch `block_num` +- runtime input shapes only change workload splitting across the fixed tasks + +The convention in the current `rope.py` is: + +```python +task_num = vec_core_num +m_num = vec_core_num // 2 + +with T.Kernel(m_num, is_npu=True) as (cid, vid): + task_id = cid * 2 + vid +``` + +This means: + +- `cid` ranges over `[0, vec_core_num // 2)` +- `vid` ranges over `[0, 2)` +- the total task count is fixed as `task_num = vec_core_num` + +As a result, `rope.py` also derives the compile-time token count for one specialization using the fixed task count: + +```python +max_rows_num_in_ub = _derive_max_rows_num_in_ub(...) +compile_num_tokens = task_num * max_rows_num_in_ub +``` + +### 3.2 Implement `generate_source(...)` + +`generate_source(...)` lowers the TileLang kernel above into the final source code. The export layer takes one specialization and turns it into compilable Ascend-C source. + +For `rope`, the core logic is: + +```python +@staticmethod +def generate_source(head_dim: int, rope_dim: int, dtype: str) -> str: + vec_core_num = detect_vec_core_num() + tilelang_kernel = build_rope_kernel( + head_dim=head_dim, + rope_dim=rope_dim, + vec_core_num=vec_core_num, + ub_buffer_bytes=FIXED_UB_BUFFER_BYTES, + ) + with tilelang.tvm.transform.PassContext(...): + kernel = tilelang.engine.lower(tilelang_kernel) + return kernel.kernel_source +``` + +The rules here are: + +- the inputs to `generate_source(...)` come from the current `SPECIALIZATIONS` entry +- `generate_source(...)` calls `build_rope_kernel(...)` +- the return value is the lowered source string + +### 3.3 Declare `DISPATCH_SCHEMA` and `SPECIALIZATIONS` + +After the kernel implementation and export layer are done, use an `@register_kernel` class to attach the kernel to the framework. + +The current minimal template in `rope.py` is: + +```python +from ....common.spec import DispatchField, TilelangKernel, register_kernel + +@register_kernel +class RopeKernel(TilelangKernel): + DISPATCH_SCHEMA = [ + DispatchField("head_dim", "int32"), + DispatchField("rope_dim", "int32"), + DispatchField("dtype", "dtype"), + ] + SPECIALIZATIONS = [ + { + "variant_key": "hd128_rd128_bf16", + "head_dim": 128, + "rope_dim": 128, + "dtype": "bf16", + }, + { + "variant_key": "hd576_rd64_bf16", + "head_dim": 576, + "rope_dim": 64, + "dtype": "bf16", + }, + ] + + @staticmethod + def generate_source(head_dim: int, rope_dim: int, dtype: str) -> str: + ... +``` + +There are two concepts to distinguish here: + +- `DISPATCH_SCHEMA` + - defines the field names, order, and types of the runtime specialization + - is the single source of truth for the C++ specialization struct, builder, and lookup interface +- `SPECIALIZATIONS` + - represents the set of instances that will actually be compiled + - each item corresponds to one variant + +The rules are: + +- every field in `DISPATCH_SCHEMA` must appear in every `SPECIALIZATIONS` item +- `SPECIALIZATIONS` may contain extra fields; those fields are passed into `generate_source(...)`, but do not enter the runtime dispatch schema +- `variant_key` is the unique identifier for that specialization +- `DISPATCH_SCHEMA` and `SPECIALIZATIONS` must match the runtime specialization one-to-one + +For `rope`, the runtime dispatch dimensions are: + +- `head_dim` +- `rope_dim` +- `dtype` + +So these three fields must appear in both: + +- `DISPATCH_SCHEMA` +- every `SPECIALIZATIONS` item + +At build time, Ascend build resolves the actual `bisheng_arch` from the `--device a2|a3` value passed by the main build path. + +### 3.4 Inspect the Generated Ascend-C Source + +When debugging the implementation details in `build_rope_kernel(...)`, or comparing how different kernel styles affect the final code generation, use the common `compile-kernels` entry to regenerate artifacts and inspect the Ascend-C source for the specialization you care about. + +For `rope`, you can fix: + +- `head_dim=576` +- `rope_dim=64` +- `dtype=bf16` + +Then regenerate the `rope` artifacts: + +```bash +python xllm/compiler/tilelang_launcher.py compile-kernels \ + --target ascend \ + --device a3 \ + --output-root /tmp/tilelang_debug \ + --kernels rope \ + --force +``` + +It is recommended to keep `--force` so the source and object files are regenerated from the current code instead of reusing an old cache hit. + +This command uses an isolated debug output directory, `/tmp/tilelang_debug`, so only the debug artifacts for `rope` are generated there and they do not get mixed with artifacts from other kernels in the main build directory. + +After that, you can directly inspect the generated source for the specialization, including the entry function, UB allocation, and vector compute logic: + +```bash +sed -n '1,200p' \ + /tmp/tilelang_debug/targets/ascend/rope/hd576_rd64_bf16/rope_hd576_rd64_bf16_kernel.cpp + +rg -n 'extern "C"|__global__|alloc_ub|alloc_shared|g_tilingKey' \ + /tmp/tilelang_debug/targets/ascend/rope/hd576_rd64_bf16/rope_hd576_rd64_bf16_kernel.cpp +``` + +To compare two kernel implementations, keep the specialization fixed, run `compile-kernels --force` before and after the change, then diff the generated `.cpp` file: + +```bash +cp /tmp/tilelang_debug/targets/ascend/rope/hd576_rd64_bf16/rope_hd576_rd64_bf16_kernel.cpp \ + /tmp/rope_before.cpp + +diff -u /tmp/rope_before.cpp \ + /tmp/tilelang_debug/targets/ascend/rope/hd576_rd64_bf16/rope_hd576_rd64_bf16_kernel.cpp +``` + +This helps isolate specialization changes from kernel implementation changes. + +After generation, the main files to inspect are: + +- `/tmp/tilelang_debug/targets/ascend/rope/hd576_rd64_bf16/rope_hd576_rd64_bf16_kernel.cpp` +- `/tmp/tilelang_debug/targets/ascend/rope/registry.inc` +- `/tmp/tilelang_debug/targets/ascend/rope/manifest.json` + +These correspond to: + +- the final Ascend-C source for one specialization +- the runtime dispatch interface directly included by the wrapper +- the full compiled artifact record for the current kernel + +The recommended debugging sequence is: + +1. run `compile-kernels --force` to regenerate the current kernel artifacts +2. inspect the `.cpp` for the specialization and analyze the code generation result +3. inspect `registry.inc` and `manifest.json` to confirm they match expectations +4. finally run `rope_wrapper_test` to check end-to-end behavior and performance + +## 4. Update the Wrapper + +When adding a new `kernel`, you need a new wrapper. When adding a new `specialization`, the wrapper only needs an update if the runtime specialization semantics change. + +For `rope_wrapper.cpp`, the manually written parts should remain: + +- tensor shape, dtype, and layout validation +- reshaping inputs into `x_rows / sin_rows / cos_rows` +- constructing the runtime specialization from tensors +- assembling launch arguments and calling `entry->fn(...)` + +### 4.1 What `registry.inc` Generates Automatically + +`registry.inc` is generated automatically from the Python-side `DISPATCH_SCHEMA`, `SPECIALIZATIONS`, and the exported Ascend-C ABI. + +For `rope`, the generated content includes: + +- `RopeSpecialization` +- `RopeHeadDim` +- `RopeRopeDim` +- `RopeDType` +- `RopeKernelFn` +- `make_rope_specialization(...)` +- `find_rope_kernel_entry(...)` +- `available_rope_variant_keys()` + +For `rope_wrapper.cpp`, `registry.inc` directly provides dispatch-related definitions such as `RopeSpecialization`, `operator==(...)`, and `RopeKernelFn`. Dtype conversion uses the shared helper `to_tilelang_dtype(...)`. + +### 4.2 What the Wrapper Actually Needs to Write + +The most important handwritten logic in `rope_wrapper.cpp` is constructing the runtime specialization from the tensors. The current code looks like this: + +```cpp +RopeSpecialization build_runtime_specialization(const torch::Tensor& x_rows) { + return make_rope_specialization( + RopeHeadDim{static_cast(x_rows.stride(0))}, + RopeRopeDim{static_cast(x_rows.size(1))}, + RopeDType{to_tilelang_dtype(x_rows.scalar_type())}); +} +``` + +For `rope`: + +- `head_dim` maps to `x_rows.stride(0)`, which is the `x_stride` used by the kernel +- `rope_dim` maps to `x_rows.size(1)` +- `dtype` maps to `x_rows.scalar_type()` + +The runtime path is: + +1. the wrapper reshapes the inputs into `x_rows / sin_rows / cos_rows` +2. `build_runtime_specialization(...)` constructs a specialization from `x_rows` +3. `find_rope_kernel_entry(...)` performs an exact match in the static registry +4. after a match, `entry->fn(...)` calls the actual compiled symbol + +The current lookup strategy is a linear scan with exact matching. If any of `head_dim`, `rope_dim`, or `dtype` differs, the lookup will miss. + +So when you add a new `specialization`, the main things to cross-check are: + +- the field semantics in Python-side `DISPATCH_SCHEMA` +- the field values in Python-side `SPECIALIZATIONS` +- the field values constructed by `build_runtime_specialization(...)` in the wrapper + +All three must match exactly. + +### 4.3 Generate and Inspect `registry.inc` First + +Before writing or modifying wrapper code, generate `registry.inc` once and inspect it. Focus on: + +- whether the generated field order in `RopeSpecialization` matches expectations +- whether the generated wrapped field type names match expectations +- the parameter order of `make_rope_specialization(...)` +- the generated entry symbol names + +`registry.inc` is the direct contract for the wrapper. Inspect it first, then write the wrapper against it. + +## 5. Update CMake + +When adding a new `kernel`, register it in `xllm/xllm/core/kernels/npu/tilelang/CMakeLists.txt`. + +CMake registration is unified through the high-level helper: + +- `tilelang_register_runtime_kernel(NAME WRAPPER_SRCS )` + +Using `rope` as the example, the minimal template is: + +```cmake +tilelang_register_runtime_kernel( + NAME rope + WRAPPER_SRCS rope_wrapper.cpp +) +``` + +This helper will: + +- derive the manifest path as `TILELANG_GENERATED_ROOT/targets/ascend//manifest.json` +- import the manifest +- add the wrapper source and compiled objects into `tilelang_kernels` +- append the `XLLM_TL__REGISTRY_INC=...` compile definition automatically + +So when adding a new runtime kernel, the CMake-side work mainly consists of two things: + +1. make sure the Python side can already generate the manifest for that kernel +2. add one `tilelang_register_runtime_kernel(...)` entry in the TileLang CMakeLists + +For day-to-day kernel additions, add one `tilelang_register_runtime_kernel(...)` line directly in CMake. `tilelang_import_kernel_manifest(...)` stays underneath as the implementation base for that higher-level helper. + +## 6. Validate + +The recommended validation order is: + +1. compile the TileLang kernel and inspect the generated `registry.inc` +2. then run the full wrapper test + +Common commands: + +```bash +python xllm/compiler/tilelang_launcher.py compile-kernels \ + --target ascend \ + --device a3 \ + --output-root build/cmake.linux-aarch64-cpython-311/xllm/compiler/tilelang \ + --kernels rope + +python setup.py test --test-name rope_wrapper_test --device a3 +``` + +The first command generates `manifest.json`, `registry.inc`, and the object files. The second command validates the full integration path. + diff --git a/docs/src/content/docs/en/features/async_schedule.md b/docs/src/content/docs/en/features/async_schedule.md new file mode 100644 index 0000000000..442bb6b37f --- /dev/null +++ b/docs/src/content/docs/en/features/async_schedule.md @@ -0,0 +1,30 @@ +--- +title: "Async schedule" +sidebar: + order: 10 +--- +## Background +The inference process of large language models can be divided into three sequential stages:1) CPU-side scheduling (preparing model inputs), 2) Device computation (GPU/TPU execution), 3) CPU-side post-processing (output handling). +Due to the sequential nature of decoding operations, the input for step-i+1 depends on the output of step-i. This forces strict serial execution of all three stages, creating device idle periods (“bubbles”) during CPU-bound stages 1 and 3, leading to suboptimal resource utilization. + +## Introduction +xLLM addresses this at the framework level by supporting asynchronous scheduling, where the CPU proactively executes scheduling operations for step-i+1 while the device is computing step-i. This allows the device to immediately begin computing step-i+1 upon completing step-i, thereby eliminating bubbles. Specifically, after initiating the computation call for step-i, the CPU does not wait for the device to finish computing. Instead, it constructs fake tokens for the step-i request, uses these fake tokens to perform scheduling operations for step-i+1 (such as allocating KV Cache), and replaces them with the true tokens computed in step-i when launching step-i+1 computation to ensure correctness. Meanwhile, the CPU processes the results of step-i in a separate thread and returns them to the client. +In the overall architecture, stages 1 and 3 on the CPU side are handled by different thread pools, and RPC function calls employ non-blocking C++ future and promise mechanisms to achieve a fully asynchronous runtime. +![Async schedule](../../assets/async_schedule_architecture.jpg) + +## Usage + +xLLM provides the gflags parameter `enable_schedule_overlap`, which defaults to false. To enable this feature, set it to true in xLLM's service startup script: +```shell +--enable_schedule_overlap=true +``` + +## Performance + +- With asynchronous scheduling enabled, the device idle time between two steps is approximately 200us - comparable to a single kernel launch duration. +- On the DeepSeek-R1-Distill-Qwen-1.5B model with TPOT constrained to 50ms, this achieves 17% throughput improvement. + +## Notice +The asynchronous scheduling feature requires the server to compute one additional step. For use cases involving limited output tokens (e.g., few-token generation) or single-output scenarios like embedding models, enabling this feature is not recommended as it may reduce server-side throughput, thus hard-disabled internally. +The VLM model is currently being adapted, will be temporarily disabled. + diff --git a/docs/src/content/docs/en/features/chunked_scheduler.md b/docs/src/content/docs/en/features/chunked_scheduler.md new file mode 100644 index 0000000000..d7222405cc --- /dev/null +++ b/docs/src/content/docs/en/features/chunked_scheduler.md @@ -0,0 +1,20 @@ +--- +title: "ChunkedPrefill Scheduler" +sidebar: + order: 21 +--- +## Feature Introduction +xLLM supports the chunked prefill scheduling strategy. Chunked prefill is a technique that optimizes large language model inference by splitting long prompts into smaller chunks for batch processing, rather than processing the entire prompt at once. +This method can effectively reduce peak GPU memory usage, improve device utilization, and better schedule and mix processing with requests from the decode stage. + +## Usage +The aforementioned strategy has been implemented in xLLM and is exposed through gflags parameters to control the feature's on/off state. + +- Enable chunked prefill and set the chunked size, if not set chunked size, its default value is equal to max_tokens_per_batch. +```bash +--enable_chunked_prefill=true +--max_tokens_per_chunk_for_prefill=20480 # optional +``` + +## Performance Impact +After enabling chunked prefill, on the Qwen3-8B model with a TPOT constraint of 50ms, the TTFT latency **decreased by 46%**. diff --git a/docs/src/content/docs/en/features/disagg_pd.md b/docs/src/content/docs/en/features/disagg_pd.md new file mode 100644 index 0000000000..2a0edad184 --- /dev/null +++ b/docs/src/content/docs/en/features/disagg_pd.md @@ -0,0 +1,87 @@ +--- +title: "Disaggregated PD" +sidebar: + order: 30 +--- +## Background +LLM online inference services typically need to meet two performance metrics: TTFT and TPOT. Traditional Contiguous Batching scheduling strategies mix Prefill and Decode requests during scheduling, causing Prefill and Decode phases to compete for computational resources. This prevents maximized utilization of computing resources and impacts performance metrics. To resolve this conflict, the Prefill and Decode phases are split to run on independent computational resources, enabling parallel execution. This simultaneously reduces TTFT and TPOT while improving throughput. + +## Introduction +The xLLM PD Separation feature is primarily implemented through the following three modules: + +- **etcd**: Stores metadata such as instance information. +- **xLLM Service**: Schedules requests and manages all computing instances. +- **xLLM**: Handles request computation instances. + +The overall architecture is shown below: +![xLLM PD Separation Architecture](../../assets/pd_architecture.jpg) + +## Usage +### Preparation +#### Install Dependencies +- **xLLM**: Refer to [Installation && Compilation](/en/getting_started/quick_start/) +- **xLLM Service**: Refer to [xLLM Service](https://github.com/xLLM-AI/xllm-service) + +#### Obtain Environment Information +Deploying Disaggregated PD Service requires obtaining the Device IP of the machine to create communication resources. Execute the command `cat /etc/hccn.conf | grep address` on the current AI Server to get the Device IP, for example: +``` +address_0=xx.xx.xx.xx +address_1=xx.xx.xx.xx +``` +`address_xx` represents the Device IP. + +### Start Disaggregated PD Service +1. Start etcd +```bash +./etcd +``` +2. Start xLLM Service +```bash +ENABLE_DECODE_RESPONSE_TO_SERVICE=true ./xllm_master_serving --etcd_addr="127.0.0.1:12389" --http_server_port 28888 --rpc_server_port 28889 --tokenizer_path=/path/to/tokenizer_config_dir/ +``` +3. Start xLLM +- Taking Qwen2-7B as an example + - Start Prefill Instance + ```bash + /path/to/xllm --model=Qwen2-7B-Instruct \ + --port=8010 \ + --devices="npu:0" \ + --master_node_addr="127.0.0.1:18888" \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_disagg_pd=true \ + --instance_role=PREFILL \ + --etcd_addr="127.0.0.1:12389" \ + --transfer_listen_port=26000 \ + --disagg_pd_port=7777 \ + --node_rank=0 \ + --nnodes=1 + ``` + - Start Decode Instance + ```bash + /path/to/xllm --model=Qwen2-7B-Instruct \ + --port=8020 \ + --devices="npu:1" \ + --master_node_addr="127.0.0.1:18898" \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_disagg_pd=true \ + --instance_role=DECODE \ + --etcd_addr="127.0.0.1:12389" \ + --transfer_listen_port=26100 \ + --disagg_pd_port=7787 \ + --node_rank=0 \ + --nnodes=1 + ``` + Important notes: + + - PD disaggregation requires reading the `/etc/hccn.conf` file. Make sure this file on the physical machine is mapped into the container. + + - `etcd_addr` must match the `etcd_addr` of `xllm_service` + +## Notice +With the chunked-prefill PD scheduler, the Prefill instance supports prefix cache. When enabled, the scheduler matches existing prefix-cache blocks before calculating the current chunk budget, so cached prompt blocks are not recomputed: +```shell +--enable_chunked_prefill=true +--enable_prefix_cache=true +``` diff --git a/docs/src/content/docs/en/features/eplb.md b/docs/src/content/docs/en/features/eplb.md new file mode 100644 index 0000000000..2a280ca556 --- /dev/null +++ b/docs/src/content/docs/en/features/eplb.md @@ -0,0 +1,33 @@ +--- +title: "MoE Load Balancing (EPLB)" +sidebar: + order: 71 +--- +## Background + +MoE models rely on dynamic token routing to distribute tokens among experts. However, in real-world deployments, uneven data distribution leads to expert load imbalance (some overloaded while others idle). Expert redundancy adjustment (e.g., adding/removing replicas) consumes additional GPU memory and may impact inference latency due to weight migration, posing significant implementation challenges. To address this, we employ an expert redundancy strategy (replicating hot experts) combined with hierarchical and global dynamic load balancing to achieve dynamic MoE load balancing. + +## Features + +The xLLM MoE Load Balancing (EPLB) functionality is implemented through three main modules: + +- **EPLB Manager**: Responsible for monitoring expert loads, collecting and managing expert distribution updates. It uses a layer-by-layer update mechanism, determining whether to update each layer based on expert load changes. +- **EPLB Executor**: The actual executor for expert distribution updates. +- **EPLB Policy**: Strategy for generating new expert load tables. + +Overall architecture diagram: +![xLLM eplb](../../assets/eplb_architecture.png) + +## Usage + +Simply add the following gflag parameters when launching xLLM: + +(Replace with actual number of devices. `ep_size` must match the number of devices) + +- xLLM provides the gflag parameter `enable_eplb` (default: false). Set to true in the xLLM service startup script to enable dynamic expert load balancing. +- `expert_parallel_degree` and `ep_size` are MoE-related parameters. `expert_parallel_degree` should be set to `2`, and `ep_size` must match the actual number of NPU/GPU devices. See [moe_params](/en/features/moe_params/) +- `eplb_update_interval` sets the expert distribution update interval in seconds (default: 1000). +- The expert distribution update uses a layer-by-layer mechanism based on expert load. When the similarity between consecutive loads for a layer is below `eplb_update_threshold`, that layer is updated (default: 1, range: 0-1). + +```bash +--enable_eplb=true --expert_parallel_degree=2 --ep_size=16 --eplb_update_interval=2000 --eplb_update_threshold=0.9 diff --git a/docs/src/content/docs/en/features/flashcomm.md b/docs/src/content/docs/en/features/flashcomm.md new file mode 100644 index 0000000000..0de3cf9bba --- /dev/null +++ b/docs/src/content/docs/en/features/flashcomm.md @@ -0,0 +1,124 @@ +--- +title: "FlashComm" +sidebar: + order: 82 +--- + +## Feature Introduction + +FlashComm is a prefill communication optimization for NPU Tensor Parallel inference in xLLM. It targets the communication overhead after row-parallel linear layers in long-prefill workloads, and uses a Matmul + ReduceScatter fused operator when the workload is suitable. + +FlashComm currently contains two parts: + +- **Sequence sharding**: during prefill, the token sequence is split across TP ranks so part of the following computation runs on local token shards. +- **MMRS fused operator**: in supported row-parallel linear layers, the normal `matmul + reduce_scatter` path is replaced by the torch_npu `npu_mm_reduce_scatter_base` operator, which fuses Matmul and ReduceScatter. + +FlashComm is disabled by default. Even when the feature flag is enabled, it only becomes active when the runtime conditions are satisfied. Otherwise, xLLM falls back to the original execution path. + +## Design + +The FlashComm flow is: + +1. When a request enters prefill, the runtime builds a FlashComm context from the token count, parallel configuration, and feature flags. +2. If the context is active, hidden states are sharded along the sequence dimension across TP ranks. +3. Supported row-parallel linear layers first try the MMRS fused path. +4. If MMRS is not applicable, for example because of unsupported shape, dtype, bias, or communication context, the caller falls back to the normal matmul and reduce_scatter path. +5. At boundaries that require full hidden states, xLLM gathers the sequence back. + +The current MMRS path uses the torch_npu `npu_mm_reduce_scatter_base` operator. The xLLM wrapper is intentionally thin: it validates inputs, resolves the HCCL group, selects `comm_mode`, and records logs. It does not reimplement the Matmul + ReduceScatter kernel. + +## Suitable Workloads + +FlashComm is most suitable for: + +- NPU backend. +- Long prefill, for example input length greater than or equal to 8K tokens. +- Large TP size. The current default activation condition recommends `TP >= 8`. +- `dp=1` and `cp=1`. +- Workloads where prefill is a large part of end-to-end latency, such as 8K/128 and 32K/1K. +- BF16/FP16 non-quantized row-parallel linear layers. + +FlashComm is usually not suitable, or has limited benefit, for: + +- Decode phase. FlashComm only optimizes prefill, so TPOT usually does not directly benefit. +- Short prompts, such as 2K input. Communication may not dominate enough to offset sharding, gather, and scheduling overhead. +- Long-output workloads where decode dominates total latency. +- `TP < 8`, `dp > 1`, or `cp > 1`, which are not enabled by the default activation rule. +- Quantized row-parallel paths. MMRS is currently wired only for the normal BF16/FP16 matmul path. + +## Usage + +Both FlashComm and the MMRS fused operator are disabled by default. For long-prefill NPU serving with TP=8 or larger, enable them explicitly: + +```bash +--enable_flashcomm1=true \ +--enable_mmrs_fusion=true \ +--flashcomm1_min_prefill_tokens=8192 \ +--mmrs_comm_mode=aiv +``` + +Graph Mode is also recommended to reduce Host-side scheduling overhead: + +```bash +--enable_graph=true \ +--enable_prefill_piecewise_graph=true +``` + +A complete recommended configuration is: + +```bash +--enable_graph=true \ +--enable_prefill_piecewise_graph=true \ +--enable_flashcomm1=true \ +--enable_mmrs_fusion=true \ +--flashcomm1_min_prefill_tokens=8192 \ +--mmrs_comm_mode=aiv +``` + +Flag reference: + +| Flag | Default | Description | +|------|---------|-------------| +| `enable_flashcomm1` | `false` | Main FlashComm switch | +| `enable_mmrs_fusion` | `false` | Enables the Matmul + ReduceScatter fused operator | +| `flashcomm1_min_prefill_tokens` | `8192` | Minimum prefill token count required before FlashComm can become active | +| `mmrs_comm_mode` | `aiv` | torch_npu MMRS communication mode. Supported values: `aiv`, `ai_cpu`, `none` | + +In most cases, keep `mmrs_comm_mode=aiv`. If some shapes hit AICore errors on the AIV path, switch temporarily to: + +```bash +--mmrs_comm_mode=ai_cpu +``` + +## Recommended Configuration + +Use the following table as the initial tuning guidance: + +| Scenario | Recommendation | +|----------|----------------| +| 8K input / short output | Enable FlashComm and MMRS | +| 32K input / medium or short output | Enable FlashComm and MMRS | +| 2K input / long output | Do not enable by default; benefits are usually unstable | +| TP=2 or TP=4 | Do not enable by default | +| TP=8 | Currently the best starting point for evaluation | +| Chunked Prefill | Can be enabled, but keep the chunk size no smaller than `flashcomm1_min_prefill_tokens`; otherwise a single chunk may not trigger FlashComm | + +If your service mixes short and long prompts, keep FlashComm disabled by default and enable it only for long-input services, long-context models, or separately deployed long-prompt workloads. + +## Performance and Correctness Notes + +- FlashComm only optimizes prefill. Focus on TTFT, prompt throughput, and prefill communication in profiling. +- TPOT, decode throughput, and long-output latency may not improve. If decode dominates, end-to-end latency may show little benefit. +- With MMRS enabled, profiling should show that part of the row-parallel communication is replaced by the Matmul + ReduceScatter fused path. +- Extra gather, layout conversion, or Host scheduling overhead can offset the MMRS benefit. +- Use multiple warmup-completed stable rounds for performance conclusions. Do not use absolute latency from profiling runs as the final benchmark number. + +## Validation Checklist + +Before enabling FlashComm in production or changing thresholds, run at least: + +1. Compare `enable_flashcomm1=false, enable_mmrs_fusion=false` against `enable_flashcomm1=true, enable_mmrs_fusion=true`. +2. Use the same model, TP size, input length, output length, and concurrency. +3. Record TTFT, TPOT, prompt throughput, decode throughput, request throughput, and latency. +4. For long-input workloads, collect profiling and verify that the MMRS path is actually hit. +5. Run a small numerical consistency check to confirm that outputs match with FlashComm on and off. diff --git a/docs/src/content/docs/en/features/global_kvcache.md b/docs/src/content/docs/en/features/global_kvcache.md new file mode 100644 index 0000000000..a12e16d20c --- /dev/null +++ b/docs/src/content/docs/en/features/global_kvcache.md @@ -0,0 +1,45 @@ +--- +title: "Global Multi-Level KV Cache" +sidebar: + order: 51 +--- +## Background +In the decoding phase of large language models (LLMs), frequent access to historical KV cache due to autoregressive generation creates a bottleneck in memory bandwidth. As model sizes and context windows expand (e.g., 128K Tokens consuming over 40GB of memory), the pressure on single-device memory increases dramatically. Existing solutions (such as vLLM) exhibit significant limitations in long-context scenarios: prefill time surges, severe memory bandwidth contention during decoding, and the need for excessive resource reservation to meet SLO requirements (TTFT < 2s, TBT < 100ms). This often results in GPU utilization below 40% and difficulties in leveraging cross-server resources. To address this, we propose a distributed global multi-level KV cache management system, adopting a memory-compute integrated architecture to break through single-machine resource constraints. + +## Feature Introduction +The xLLM Global KV Cache feature is primarily implemented through the following three modules: +- **etcd**: For cluster service registration, load information synchronization, and global cache state management. +- **xLLM Service**: For scheduling requests and managing all compute instances. +- **xLLM**: The compute instances handling requests. + +The overall architecture is shown in the diagram below: +![xLLM Global Multi-Level KV Cache](../../assets/globalkvcache_architecture.png) + +## Usage Example + +### Preparation + +#### Install Dependencies +- **xLLM**: Refer to [Quick Start](/en/getting_started/quick_start/) +- **xLLM Service**: Refer to [xLLM Service](https://github.com/xLLM-AI/xllm-service) + +### Usage Instructions + +1. **etcd Startup Configuration:** + ```bash + ./etcd --listen-peer-urls=http://0.0.0.0:10999 --listen-client-urls=http://0.0.0.0:10998 + ``` + +2. **xLLM Service Startup Configuration:** + ```bash + ./xllm_master_serving --etcd_addr="127.0.0.1:10998" --http_server_port 28888 --rpc_server_port 28889 --tokenizer_path=/path/to/tokenizer_config_dir/ + ``` + +3. **xLLM Startup Configuration:** + Add the following gflag parameters when starting xLLM: + ```bash + --enable_service_routing=true + --enable_cache_upload=true + # PD separation currently does not support Global KVCache Management + --enable_disagg_pd=false + ``` diff --git a/docs/src/content/docs/en/features/graph_mode.md b/docs/src/content/docs/en/features/graph_mode.md new file mode 100644 index 0000000000..36557b43ed --- /dev/null +++ b/docs/src/content/docs/en/features/graph_mode.md @@ -0,0 +1,84 @@ +--- +title: "Graph Mode" +sidebar: + order: 81 +--- +## Overview + +xLLM supports Graph Mode: computation graphs are pre-captured and replayed in subsequent runs to reduce CPU overhead and improve inference performance. Graph Mode has corresponding implementations on different hardware platforms. + +## Feature Description + +To optimize Host-side scheduling, graph mode submits a large task from the CPU once and then executes small kernels in a streaming manner on the device, significantly reducing startup time and device bubbles. + +In the xLLM engine, Graph Mode provides the following: + +### Dynamic Shape Parameterization + - Key dynamic dimensions other than num_tokens are treated as whole-graph input parameters, including batch_size, kv_seq_lens, q_seq_lens, block_table_size, and the like, for flexibility. During memory allocation and kernel configuration. At graph launch, the actual values of these parameters are passed so kernels use the correct strides to access data. + +### Piecewise Graph + - When some operators do not support graph capture and thus break the full graph, each segment (piece) after the break is captured as a separate graph. This maximizes graph-mode benefits even when the full graph cannot be captured, and is commonly used for prefill and chunked prefill. + +### Multi-Shape Reusable Memory Pool + - To avoid waste from separate memory buffers (input, output, and intermediate tensors) per shape, we use an expandable memory pool. Multiple shapes share the pool base address, with different shapes using different offsets from that base. + +## Usage + +These capabilities are implemented inside the xLLM engine and are generally controlled through gflags. + +The minimal configuration only needs `enable_graph` to turn on Graph Mode for the decode phase: + +```shell +--enable_graph=true +``` + +Common companion flags include: + +- `enable_graph`: enables the base Graph Mode capability for the decode phase +- `enable_prefill_piecewise_graph`: enables Piecewise Graph for the prefill phase +- `enable_graph_mode_decode_no_padding`: builds decode graphs with the actual `num_tokens` instead of the padded shape +- `max_tokens_for_graph_mode`: limits the maximum number of tokens covered by Graph Mode; `0` means no limit + +If you want to enable both decode Graph and prefill Piecewise Graph, use: + +```shell +--enable_graph=true \ +--enable_prefill_piecewise_graph=true \ +--max_tokens_for_graph_mode=2048 +``` + +If you need decode graph capture without padding, add: + +```shell +--enable_graph=true \ +--enable_graph_mode_decode_no_padding=true +``` + +For a more complete description of the flags, see [CLI Reference](/en/cli_reference/). + +## Performance Impact + +- With Graph Mode enabled, decode-phase throughput **improves by about 8%–10%** on models such as Qwen3-0.6B and Qwen3-1.7B. + +## Model Support + +The following table lists each model’s support on ACLGraph, CudaGraph, and MLUGraph. + +| Model | ACLGraph | CudaGraph | MLUGraph | +|------|----------|-----------|----------| +| Qwen3/Qwen3-MoE | ✅ | ✅ | ✅ | +| DeepseekV3.2 | ✅ | | ✅ | +| GLM4.5/4.6/4.7 | ✅ | | | +| Qwen2.5-VL | | | ✅ | +| Qwen3-VL/Qwen3-VL-MoE | ✅ | | | +| GLM4V | ✅ | | | +| GLM4V-MoE | ✅ | | | + +:::caution[Adding Graph Mode support for new models] +Ensure that the kernels used in the computation implement dynamic dimension parameterization; otherwise the graph may break and kernels may need to be re-implemented. + +::: +## Related Documentation + +- For more detailed Graph Mode design and implementation notes, including ACL Graph / CUDA Graph fundamentals, dynamic dimension parameterization, Piecewise Graph, and multi-shape memory reuse, see: [Graph Mode Design Document](/en/design/graph_mode_design/) + diff --git a/docs/src/content/docs/en/features/moe_params.md b/docs/src/content/docs/en/features/moe_params.md new file mode 100644 index 0000000000..521a795ba0 --- /dev/null +++ b/docs/src/content/docs/en/features/moe_params.md @@ -0,0 +1,28 @@ +--- +title: "EP Parallelism" +sidebar: + order: 70 +--- +## Background + +When deploying the DeepSeek-R1 671B parameter-scale model, traditional distributed deployment faces core bottlenecks including low GPU memory utilization, high communication overhead, and expensive hardware costs. Therefore, Expert Parallelism (EP) is introduced. + +Key advantages: ++ With the same resources, fewer Experts per GPU means more memory available for KV Cache, allowing more tokens to be cached. ++ Due to MLA characteristics, smaller TP Size with the same resources means less redundant KV Cache, enabling more tokens to be cached. ++ Large-scale EP deployment can concentrate token computations for the same expert on the same device, improving hardware utilization. + +## Parameter Configuration + ++ **dp_size**: Sets the data parallelism scale for the Attention part. Default: 1, can be set to powers of 2. When dp_size doesn't equal the total number of devices, TP parallelism is used within the DP group. ++ **ep_size**: Sets the expert parallelism scale for the MoE part. Default: 1, can be set to powers of 2. When ep_size doesn't equal the total number of devices, TP parallelism is used within the DP group. ++ **expert_parallel_degree**: EP-related parameter. Default is 0 when EP is disabled. When EP is enabled, default is 1 (EP Level 1). When ep_size equals the total number of devices, can be set to 2 to enable EP Level 2. + +## Solution Design + ++ When EP is enabled (default EP Level 1), after computing both Attention and MoE parts, data is sent to the next stage via All Gather communication across all devices. Example for 64 devices (Attention: dp32tp2, MoE: ep32tp2): +![Alt text](../../assets/moe_eplevel1.jpg) + ++ When ep_size equals the total number of devices, EP Level 2 can be enabled. Communication between Attention and MoE parts changes to ALL2ALL, sending data only to required devices to reduce communication volume and overhead. Example for 64 devices: +![Alt text](../../assets/moe_eplevel2.jpg) + diff --git a/docs/src/content/docs/en/features/mtp.md b/docs/src/content/docs/en/features/mtp.md new file mode 100644 index 0000000000..f4d7ea5f17 --- /dev/null +++ b/docs/src/content/docs/en/features/mtp.md @@ -0,0 +1,323 @@ +--- +title: "MTP Speculative Inference" +sidebar: + order: 80 +--- +## Background +MTP (Multi-Token Prediction) is an innovative inference acceleration technique that addresses efficiency bottlenecks in large language model generation. By incorporating specialized pre-training designs, MTP provides efficient draft token prediction capabilities during inference, significantly improving generation speed. Its core value lies in balancing inference efficiency with output quality, offering an optimal solution for long-sequence generation problems in LLMs, ultimately optimizing inference performance. + +## Key Features +MTP offers the following core acceleration capabilities: + +- **Efficient Draft Generation**: Uses a lightweight MTP architecture to rapidly generate draft tokens that serve as input for the main model's verification, dramatically reducing computation overhead compared to traditional autoregressive generation. + +- **Batch Verification Mechanism**: The main model can simultaneously verify multiple MTP-generated draft tokens in batch, rather than processing them sequentially, significantly boosting inference speed. + +- **High Sampling Accuracy**: MTP solves the critical pain point of low token acceptance rates in post-training draft modules (like Eagle and Medusa). By optimizing draft generation during pre-training, MTP produces tokens with higher accuracy, reducing the verification burden on the main model. + +- **Reduced Inference Latency**: By pre-generating multiple potential subsequent tokens, MTP effectively decreases cumulative latency during long-text generation, creating a smoother user experience. + +- **Optimized Resource Consumption**: Compared to other inference acceleration techniques, MTP maintains acceleration effects while requiring fewer additional computational resources, making it suitable for deployment in resource-constrained environments. + +MTP technology provides a novel efficiency optimization solution for LLM inference, particularly well-suited for real-time applications requiring rapid responses, representing an important direction in language model inference optimization. + +:::note[Model Support] +Currently supports MTP structure export for the following models: +- DeepSeek-V3 (input model_type: deepseek_v3, exported MTP model_type: deepseek_v3_mtp) +- DeepSeek-V3.2 (input model_type: deepseek_v3, exported MTP model_type: deepseek_v32_mtp) +- DeepSeek-R1 (input model_type: deepseek_v3, exported MTP model_type: deepseek_v3_mtp) +- GLM4 MoE (e.g., GLM-4.5-Air, exported MTP model_type: glm4_moe_mtp) + +Note: +- DeepSeek V3 and R1 both have input model_type "deepseek_v3", and the exported MTP model will have model_type "deepseek_v3_mtp" +- DeepSeek V3.2 has input model_type "deepseek_v3" (but can be auto-detected by index_head_dim fields), and the exported MTP model will have model_type "deepseek_v32_mtp" + +::: +## Usage Example + +This example assumes the base model is not quantized. For exporting a draft model from a quantized base model, follow this link: [Exporting a draft model from a quantized model](#exporting-a-draft-model-from-a-quantized-model) + +### Export Model + +The script will automatically detect the model type, or you can manually specify it. + +#### DeepSeek-V3 +```bash +python3 tools/export_mtp.py \ + --input-dir /path/to/DeepSeek-V3 \ + --output-dir /path/to/DeepSeek-V3-mtp +``` + +#### DeepSeek-V3.2 +```bash +python3 tools/export_mtp.py \ + --input-dir /path/to/DeepSeek-V3.2 \ + --output-dir /path/to/DeepSeek-V3.2-mtp +``` + +#### DeepSeek-R1 +```bash +python3 tools/export_mtp.py \ + --input-dir /path/to/DeepSeek-R1 \ + --output-dir /path/to/DeepSeek-R1-mtp +``` + +#### GLM4 MoE +```bash +python3 tools/export_mtp.py \ + --input-dir /path/to/GLM-4.5-Air \ + --output-dir /path/to/GLM-4.5-Air-mtp +``` + +#### Manually Specify Model Type +If auto-detection fails, you can manually specify the model type: +```bash +python3 tools/export_mtp.py \ + --input-dir /path/to/model \ + --output-dir /path/to/model-mtp \ + --model-type deepseek_v3 # Options: deepseek_v3 (for V3/R1), deepseek_v32 (for V3.2), glm4_moe +``` + +Input model references: +- [DeepSeek-V3](https://huggingface.co/deepseek-ai/DeepSeek-V3) +- [DeepSeek-V3.2](https://huggingface.co/deepseek-ai/DeepSeek-V3.2) +- [DeepSeek-R1](https://huggingface.co/deepseek-ai/DeepSeek-R1) +- [GLM-4.5-Air](https://huggingface.co/zai-org/GLM-4.5-Air) + +### Launch Script + +When using MTP for inference, you need to specify both the main model and the draft model (MTP model). + +#### DeepSeek-V3/V3.2/R1 Launch Example +```bash +MODEL_PATH="/models/DeepSeek-V3" +DRAFT_MODEL_PATH="/models/DeepSeek-V3-mtp" +MASTER_NODE_ADDR="127.0.0.1:42123" +START_PORT=13222 +START_DEVICE=0 +LOG_DIR="log" +NNODES=16 + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + nohup ./xllm \ + --model $MODEL_PATH \ + --devices="npu:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --draft_model $DRAFT_MODEL_PATH \ + --draft_devices="npu:$DEVICE" \ + --num_speculative_tokens 1 \ + --max_memory_utilization=0.90 \ + --max_tokens_per_batch=10000 \ + --max_seqs_per_batch=256 \ + --block_size=128 \ + --ep_size=1 \ + --dp_size=1 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --node_rank=$i > $LOG_FILE 2>&1 & + sleep 0.5 +done +``` + +#### GLM4 MoE Launch Example +```bash +MODEL_PATH="/models/GLM-4.5-Air" +DRAFT_MODEL_PATH="/models/GLM-4.5-Air-mtp" +# ... same other configurations +``` + +## Performance Data +Based on ShareGPT dataset with input length=2500, output length=1500, total requests=80. + +| method | Concurrency | Mean TPOT(ms) | Mean TTFT(ms) | Output Tokens/s | Total Tokens/s | +|:---------:|:-----------:|:-------------:|:-------------:|:---------------:|:--------------:| +| baseline | 1 | 40.61 | 141.80 | 24.20 | 65.77 | +| mtp | 1 | 28.33 | 142.35 | 35.19 | 95.52 | +| baseline | 2 | 42.69 | 178.59 | 45.16 | 122.74 | +| mtp | 2 | 29.81 | 187.97 | 64.75 | 175.78 | +| baseline | 4 | 46.18 | 172.34 | 79.83 | 216.96 | +| mtp | 4 | 33.54 | 194.22 | 111.18 | 301.81 | +| baseline | 8 | 53.16 | 181.49 | 110.68 | 300.81 | +| mtp | 8 | 40.99 | 203.37 | 154.46 | 419.34 | +| baseline | 16 | 68.50 | 213.89 | 143.81 | 390.84 | +| mtp | 16 | 57.04 | 254.99 | 201.89 | 548.04 | +| baseline | 20 | 74.72 | 228.80 | 154.77 | 420.65 | +| mtp | 20 | 61.73 | 264.34 | 206.24 | 559.84 | +| baseline | 40 | 119.68 | 559.32 | 180.22 | 489.80 | +| mtp | 40 | 105.70 | 544.54 | 252.91 | 686.74 | +| baseline | 80 | 180.89 | 2996.21 | 192.09 | 522.06 | +| mtp | 80 | 152.19 | 2163.72 | 278.07 | 755.12 | + +## Exporting a draft model from a quantized model + +Let's assume we have downloaded a quantized Deepseek-V3-w8a8 model. Unfortunately, if we extract the draft model, this will not be quantized by default; we need to apply quantization once extracted. Here are the steps: + +### Export to a temporary draft model + +```bash +python3 tools/export_mtp.py --input-dir /path/to/DeepSeek-V3-w8a8 --output-dir /path/to/DeepSeek-V3-w8a8-temp +``` + +### Patch the temporary draft model's `config.json` file + +- Open the file and change: `"model_type": "deepseek_v3_mtp"` to `"model_type": "deepseek_v3"`. +- Remove the `"quantization_config"` entry. + + +### Patch the temporary draft model + +```bash +cd /path/to/DeepSeek-V3-w8a8-temp + +wget https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/configuration_deepseek.py + +wget https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/modeling_deepseek.py +``` + + +### Fix broken indexes + +```bash +rm -f *.index.json + +mv mtp_layer_parameters.safetensors model.safetensors + +cat << 'EOF' > /path/to/workspace/make_index.py +import json +from safetensors import safe_open + +model_dir = '/path/to/DeepSeek-V3-w8a8-temp' +tensor_file = f'{model_dir}/model.safetensors' + +weight_map = {} +# Open the safetensors file and map every tensor inside it to this file +with safe_open(tensor_file, framework="pt", device="cpu") as f: + for key in f.keys(): + weight_map[key] = "model.safetensors" + +# Build the JSON structure both libraries demand +index_data = { + "metadata": {"total_size": 0}, + "weight_map": weight_map +} + +# Save it +with open(f'{model_dir}/model.safetensors.index.json', 'w') as f: + json.dump(index_data, f, indent=2) + +print("Perfect index file created successfully!") +EOF + +python3 /path/to/workspace/make_index.py +``` + +### Install Ascend's ModelSlim toolkit for quantization + +```bash +git clone https://gitcode.com/Ascend/msit.git + +bash install.sh +``` + +### Patch ModelSlim + +```bash +sed -i 's/patch("transformers.modeling_utils.set_initialized_submodules"/# patch("transformers.modeling_utils.set_initialized_submodules"/g' /path/to/msit/msmodelslim/example/DeepSeek/quant_deepseek_w8a8.py +``` + +### Generate a quantized draft model from the temporary draft model using ModelSlim + +```bash +cd /path/to/msit/msmodelslim/example/DeepSeek + +python3 quant_deepseek_w8a8.py --model_path /path/to/DeepSeek-V3-w8a8-temp --save_path /path/to/DeepSeek-V3-w8a8-mtp --batch_size 4 --trust_remote_code True +``` + +### Patch the quantized draft model's `config.json` file + +- Open the file and change: `"model_type": "deepseek_v3"` to `"model_type": "deepseek_v3_mtp"`. +- Add the entry `"torch_dtype": "bfloat16",` if missing. + +### Rescue the quantized draft model weights + +```bash +cat << 'EOF' > /path/to/workspace/rescue_mtp.py +import json +import glob +import os +from safetensors import safe_open +from safetensors.torch import save_file + +if len(sys.argv) < 3: + print("Usage: python3 rescue_mtp.py ") + sys.exit(1) + +orig_dir = sys.argv[1] +quant_dir = sys.argv[2] + +# Find original tensor file +orig_tensor_file = f'{orig_dir}/model.safetensors' +if not os.path.exists(orig_tensor_file): + orig_tensor_file = f'{orig_dir}/mtp_layer_parameters.safetensors' + +# Find quantized index file +index_files = glob.glob(f'{quant_dir}/*.index.json') +if not index_files: + print("Error: Could not find index file in quantized directory.") + exit(1) +quant_index_file = index_files[0] + +# Load original keys +orig_keys = set() +with safe_open(orig_tensor_file, framework="pt", device="cpu") as f: + orig_keys = set(f.keys()) + +# Load quantized keys +with open(quant_index_file, 'r') as f: + quant_index = json.load(f) +quant_keys = set(quant_index['weight_map'].keys()) + +# Find the ones msmodelslim dropped +missing_keys = orig_keys - quant_keys +print(f"Rescuing {len(missing_keys)} missing weights: {missing_keys}") + +if missing_keys: + # Extract them from the original file + missing_tensors = {} + with safe_open(orig_tensor_file, framework="pt", device="cpu") as f: + for key in missing_keys: + missing_tensors[key] = f.get_tensor(key) + + # Save them into the quantized folder + out_file = "missing_mtp_weights.safetensors" + save_file(missing_tensors, f"{quant_dir}/{out_file}") + + # Update the JSON map so xllm can find them + for key in missing_keys: + quant_index['weight_map'][key] = out_file + + with open(quant_index_file, 'w') as f: + json.dump(quant_index, f, indent=2) + + print("Rescue complete! The MTP model is now whole.") +else: + print("No missing keys found. Something else is wrong.") +EOF + +# Run the script +python3 /path/to/workspace/rescue_mtp.py +``` + +### Delete the temporary draft model + +```bash +rm -rf /path/to/DeepSeek-V3-w8a8-temp +``` + +Now we can go back to starting the server: [Launch Script](#launch-script). diff --git a/docs/src/content/docs/en/features/multi_streams.md b/docs/src/content/docs/en/features/multi_streams.md new file mode 100644 index 0000000000..c1d3c61422 --- /dev/null +++ b/docs/src/content/docs/en/features/multi_streams.md @@ -0,0 +1,30 @@ +--- +title: "Multi-stream parallel" +sidebar: + order: 11 +--- +## Background +In distributed inference scenarios for large-scale models, additional communication operations are required to aggregate computation results from different devices. Taking large-scale MoE models like Deepseek as an example, the distributed scale is typically substantial, leading to increased communication overhead. + +If both computation and communication are performed on the same stream, the device’s computing resources will remain idle while waiting for communication to complete, resulting in wasted computational capacity before subsequent calculations can begin. + +## Introduction +xLLM implements multi-stream parallelism at the model layer, where the input batch is split into 2 micro-batches. One stream handles computation for the first micro-batch, another concurrently executes communication for the second micro-batch. +This overlap of computation and communication effectively hides the communication latency.![multi_streams_parallel](../../assets/multi_streams_architecture.jpg) + +## Usage + +xLLM provides the gflags parameter `enable_multi_stream_parallel`, which defaults to false. To enable this feature, set it to true in xLLM’s service startup script, as: +```shell +--enable_multi_stream_parallel=true +``` + +## Performance +With prefill dual-stream parallelism enabled, it can effectively mask over 75% of communication overhead. +On the DeepSeek-R1 model, when generating just 1 token, this achieves: +- **7%** reduction in TTFT. +- **7%** throughput improvement. + +## Notice +The dual-stream parallelism currently only supports the prefill phase, with greater performance benefits observed for longer input requests. +Only Support DeepSeek, Qwen3 dense(non-MoE) models. diff --git a/docs/src/content/docs/en/features/multimodal.md b/docs/src/content/docs/en/features/multimodal.md new file mode 100644 index 0000000000..4bcc591f65 --- /dev/null +++ b/docs/src/content/docs/en/features/multimodal.md @@ -0,0 +1,23 @@ +--- +title: "Multimodal Support" +sidebar: + order: 60 +--- +This document introduces the current multimodal support in the xLLM inference engine, including supported models, modality types, and offline and online interfaces. + +## Supported Models +- Qwen2.5-VL: including 7B/32B/72B. +- Qwen3-VL: including 2B/4B/8B/32B. +- Qwen3-VL-MoE: including A3B/A22B. +- MiniCPM-V-2_6: 7B. + +## Modality Types +- Images: supports single-image and multi-image inputs, image + prompt combinations, and text-only prompts. + +:::caution[Notes] +- The multimodal backend does not currently support prefix cache or chunked prefill. Support is in progress. +- xLLM now renders ChatTemplate uniformly based on JinJa. When deploying MiniCPM-V-2_6, the model directory must provide a ChatTemplate file. +- Image inputs support Base64 data and image URLs. +- Multimodal models currently mainly support the image modality. Video, audio, and other modalities are in progress. + +::: diff --git a/docs/src/content/docs/en/features/prefix_cache.md b/docs/src/content/docs/en/features/prefix_cache.md new file mode 100644 index 0000000000..83e6601ade --- /dev/null +++ b/docs/src/content/docs/en/features/prefix_cache.md @@ -0,0 +1,35 @@ +--- +title: "Prefix Cache Optimization" +sidebar: + order: 50 +--- +## Feature Introduction +xLLM supports prefix cache matching. The prefix cache is based on `murmur_hash` and uses an LRU eviction policy, delivering superior matching efficiency and increased prefix cache hit rates. +Additionally, the prefix cache has been optimized to support the `continuous_scheduler`, `chunked_scheduler`, and `zero_evict_scheduler`. The cache is updated immediately after prefill operations, enhancing matching timeliness. For the `chunked_scheduler`, multi-stage chunked prefill matching is supported, reducing computational overhead and minimizing KV cache usage as much as possible. + +## Usage +The prefix cache is implemented in xLLM and exposed through gflags parameters to control its functionality. + +- Enable prefix cache with specific policy and settings: +``` +--enable_prefix_cache=true +``` + +## Cache-Aware DP Routing + +When running with data parallelism (`dp_size > 1`), requests can be routed to the DP rank that holds the longest prefix-cache hit, improving KV cache reuse across ranks. + +- Enable cache-aware DP routing: +``` +--enable_prefix_cache=true --enable_prefix_cache_aware_dp_routing=true +``` + +- `prefix_cache_aware_dp_match_threshold` (default `0.5`): minimum prefix block hit ratio to prefer the cache-affinity rank. Below this threshold, routing falls back to free-block balancing. +- `prefix_cache_aware_dp_imbalance_threshold` (default `0.1`): maximum cross-rank KV utilization gap (`(max_used-min_used)/total_blocks`). Exceeding this disables affinity routing and selects the least-loaded rank. + +## Performance Impact +After enabling prefix cache, on the Qwen3-8B model with a TPOT constraint of 50ms, the E2E latency **decreased by 10%**. + +:::note +For disaggregated PD, prefix cache is supported in specific scheduler roles. See [Disaggregated PD](/en/features/disagg_pd/) for the supported configurations. +::: diff --git a/docs/src/content/docs/en/features/xllm_service_overview.md b/docs/src/content/docs/en/features/xllm_service_overview.md new file mode 100644 index 0000000000..0f7ecce7f1 --- /dev/null +++ b/docs/src/content/docs/en/features/xllm_service_overview.md @@ -0,0 +1,54 @@ +--- +title: "xLLM Service" +sidebar: + order: 90 +--- +[:simple-github: xLLM Service](https://github.com/xLLM-AI/xllm-service) + +## Project Overview + +**xLLM-service** is a service-layer framework developed based on the **xLLM** inference engine, providing efficient, fault-tolerant, and flexible LLM inference services for clustered deployment. + +xLLM-service targets to address key challenges in enterprise-level service scenarios: + +- How to ensure the SLA of online services and improve resource utilization of offline tasks in a hybrid online-offline deployment environment. + +- How to react to changing request loads in actual businesses, such as fluctuations in input/output lengths. + +- Resolving performance bottlenecks of multimodal model requests. + +- Ensuring high reliability of computing instances. + +#### Background + +LLM with parameter scales ranging from tens of billions to trillions are being rapidly deployed in core business scenarios such as intelligent customer service, real-time recommendation, and content generation. Efficient support for domestic computing hardware has become a core requirement for low-cost inference deployment. Existing inference engines struggle to effectively adapt to the architectural characteristics of dedicated accelerators like domestic chips. Performance issues such as low utilization of computing units, load imbalance and communication overhead bottlenecks under the MoE architecture, and difficulties in kv cache management have restricted the efficient inference of requests and the scalability of the system. The xLLM-service + xLLM inference engine improves the efficiency of the entire performance link and it provides crucial technical support for the large-scale implementation of LLM in real-world business scenarios. + +--- + +## Overall Architecture +The overall architecture of xLLM-service is shown in the figure below: + +![1](../../assets/service_arch.png) + +## Core Components + +### ETCD Cluster +It is used for metadata management, including the storage and management of metadata such as models, xllm instances, and requests. It also provides xllm node registration and discovery services. + +### Fault Tolerance +xLLM-service provides fault tolerance management to ensure service quality and stability. + +### Global Scheduler +It implements globally aware scheduling. Based on the current system status, it accurately dispatches requests to the optimal instances for execution, effectively improving the overall service response efficiency and resource utilization. + +### Global KV Cache Manager +It is responsible for global KV Cache management. Its core capabilities include distributed KV cache awareness, Prefix matching, and dynamic migration of KV Cache, which optimize the efficiency of cache resource usage. + +### Instance Manager +It focuses on the full-lifecycle management of instances. All xllm instances must register to service after startup. Based on preset policies, the module provides support for instances such as scheduling adaptation and fault tolerance handling. + +### Event Plane +As the metrics and event hub, it receives Metrics data reported by various instances, uniformly collects and organizes statistical indicators, and provides data support for decisions such as service scheduling, fault tolerance, and scaling. + +### Planner +It undertakes the functions of strategy analysis and decision-making. Based on the Metrics data reported by the Event Plane (including instance runtime indicators, machine load indicators, etc.), it analyzes the service scaling needs and the necessity of expanding hot instances, and outputs resource adjustment and instance optimization strategies. diff --git a/docs/src/content/docs/en/features/zero_evict_scheduler.md b/docs/src/content/docs/en/features/zero_evict_scheduler.md new file mode 100644 index 0000000000..90ccf30311 --- /dev/null +++ b/docs/src/content/docs/en/features/zero_evict_scheduler.md @@ -0,0 +1,20 @@ +--- +title: "Zero Evict Scheduler" +sidebar: + order: 22 +--- +## Feature Introduction +xLLM supports the zero evict scheduling strategy. The zero evict scheduling strategy is an algorithm designed to minimize request eviction rates, reducing the need for prefill computation on evicted requests and consequently improving TPOT (Time Per Output Token). +This scheduling algorithm employs simulation rounds to detect whether a request can be scheduled without causing the eviction of other requests. + +## Usage +The aforementioned strategy has been implemented in xLLM and is exposed through gflags parameters to control the feature's on/off state. + +- Enable the zero evict strategy and set the maximum decode tokens per sequence. +``` +--use_zero_evict=true +--max_decode_token_per_sequence=256 +``` + +## Performance Impact +After enabling zero evict, on the Qwen3-8B model with an E2E latency constraint, the TPOT latency **decreased by 27%**. diff --git a/docs/src/content/docs/en/getting_started/launch_xllm.md b/docs/src/content/docs/en/getting_started/launch_xllm.md new file mode 100644 index 0000000000..028a6398fb --- /dev/null +++ b/docs/src/content/docs/en/getting_started/launch_xllm.md @@ -0,0 +1,243 @@ +--- +title: "Launch xllm" +sidebar: + order: 3 +--- +Taking Qwen3 as an example, the script for launching xllm is as follows. The provided script is suitable for both single-node single-device and single-node multi-device scenarios. When using multiple devices on a single node, you need to modify `NNODES` (one device represents one node), as well as environment variables such as `ASCEND_RT_VISIBLE_DEVICES`, `CUDA_VISIBLE_DEVICES`, or `MLU_VISIBLE_DEVICES`. + +## NPU + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +export ASCEND_RT_VISIBLE_DEVICES=0 +export HCCL_IF_BASE_PORT=43432 # HCCL communication base port + +MODEL_PATH="/path/to/model/Qwen3-8B" # Model path +MASTER_NODE_ADDR="127.0.0.1:9748" # Master node address (must be globally consistent) +START_PORT=18000 # Service starting port +START_DEVICE=0 # Starting logical device number +LOG_DIR="log" # Log directory +NNODES=1 # Number of nodes (current script launches 1 process) + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="npu:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --max_memory_utilization=0.86 \ + --block_size=128 \ + --communication_backend="hccl" \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=true \ + --enable_schedule_overlap=true \ + --enable_shm=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +## NVIDIA GPU + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export CUDA_VISIBLE_DEVICES=0 +# for debug +# export CUDA_LAUNCH_BLOCKING=1 + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="cuda:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --block_size=32 \ + --max_memory_utilization=0.8 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_schedule_overlap=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +## MLU + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export MLU_VISIBLE_DEVICES=0 + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="mlu:$DEVICE" \ + --port $PORT \ + --nnodes=$NNODES \ + --master_node_addr=$MASTER_NODE_ADDR \ + --block_size=16 \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +## Hygon DCU + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export HIP_VISIBLE_DEVICES=0 + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="dcu:$DEVICE" \ + --port $PORT \ + --nnodes=$NNODES \ + --master_node_addr=$MASTER_NODE_ADDR \ + --block_size=128 \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +## MetaX MACA + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export CUDA_VISIBLE_DEVICES=0 +export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/data/aot/ + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="cuda:$DEVICE" \ + --port $PORT \ + --nnodes=$NNODES \ + --master_node_addr=$MASTER_NODE_ADDR \ + --block_size=128 \ + --max_memory_utilization=0.86 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_schedule_overlap=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +## Mthreads MUSA + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export MUSA_VISIBLE_DEVICES=0 + +MODEL_PATH="/path/to/model/Qwen3.5-27B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="musa:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --block_size=64 \ + --max_memory_utilization=0.8 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=true \ + --enable_schedule_overlap=true \ + --enable_graph=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +See [Mthreads MUSA](/en/hardware/musa/) for details. diff --git a/docs/src/content/docs/en/getting_started/multi_machine.md b/docs/src/content/docs/en/getting_started/multi_machine.md new file mode 100644 index 0000000000..199742acef --- /dev/null +++ b/docs/src/content/docs/en/getting_started/multi_machine.md @@ -0,0 +1,113 @@ +--- +title: "Multi-Node Deployment" +sidebar: + order: 5 +--- +This example demonstrates how to launch a 32-GPU (NPU) deployment across 2 machines. +Launching Services on the First Machine: +```shell +bash start_deepseek_machine_1.sh +``` + +The start_deepseek_machine_1.sh script is as follows: +```bash +#!/bin/bash +set -e + +rm -rf core.* + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +export HCCL_IF_BASE_PORT=43432 # HCCL communication base port + +# 4. Start distributed service +MODEL_PATH="/path/to/your/DeepSeek-R1" # Model path +MASTER_NODE_ADDR="123.123.123.123:9748" # Master node address (must be globally consistent) +LOCAL_HOST=123.123.123.123 # Local IP for service launch +START_PORT=18000 # Service starting port +START_DEVICE=0 # Starting NPU logical device number +LOCAL_NODES=16 # Number of local processes (this script launches 16 processes) +LOG_DIR="log" # Log directory +NNODES=32 # Total number of GPUs/NPUs (32 in this 2-machine example) + +mkdir -p $LOG_DIR + +for (( i=0; i<$LOCAL_NODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + /path/to/xllm \ + --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --max_memory_utilization=0.86 \ + --max_tokens_per_batch=40000 \ + --max_seqs_per_batch=256 \ + --block_size=128 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --communication_backend="hccl" \ + --enable_schedule_overlap=true \ + --rank_tablefile=./ranktable_2s_32p.json \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +Launching Services on the Second Machine: +```shell +bash start_deepseek_machine_2.sh +``` + +The start_deepseek_machine_2.sh script is as follows: +```bash +#!/bin/bash +set -e + +rm -rf core.* + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +export HCCL_IF_BASE_PORT=43432 # HCCL communication base port + +MODEL_PATH="/path/to/your/DeepSeek-R1" # Model path +MASTER_NODE_ADDR="123.123.123.123:9748" # Master node address (must be globally consistent) +LOCAL_HOST=456.456.456.456 # Local IP for service launch +START_PORT=18000 # Service starting port +START_DEVICE=0 # Starting NPU logical device number +LOCAL_NODES=16 # Number of local processes (this script launches 16 processes) +LOG_DIR="log" # Log directory +NNODES=32 # Total number of GPUs/NPUs (32 in this 2-machine example) + +mkdir -p $LOG_DIR + +for (( i=0; i<$LOCAL_NODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + /path/to/xllm \ + --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --max_memory_utilization=0.86 \ + --max_tokens_per_batch=40000 \ + --max_seqs_per_batch=256 \ + --block_size=128 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --communication_backend="hccl" \ + --enable_schedule_overlap=true \ + --rank_tablefile=./ranktable_2s_32p.json \ + --node_rank=$((i + LOCAL_NODES)) \ > $LOG_FILE 2>&1 & +done +``` +This example uses 2 machines. You can set the total number of GPUs/NPUs via `--nnodes`, where `--node_rank` specifies the global rank ID for each node. +The `--rank_tablefile=./ranktable_2s_32p.json`parameter points to the configuration file required for establishing the NPU communication domain. For instructions on generating this file, refer to [Ranktable Generation](https://gitee.com/mindspore/models/blob/master/utils/hccl_tools/README.md). + diff --git a/docs/src/content/docs/en/getting_started/offline_service.md b/docs/src/content/docs/en/getting_started/offline_service.md new file mode 100644 index 0000000000..5bc87cf513 --- /dev/null +++ b/docs/src/content/docs/en/getting_started/offline_service.md @@ -0,0 +1,43 @@ +--- +title: "Offline Inference" +sidebar: + order: 7 +--- +To facilitate users in quickly using xLLM for offline inference, we provide Python script examples for launching offline inference. + +## LLM + +LLM inference example: [:simple-github: https://github.com/xLLM-AI/xllm/blob/main/examples/generate.py](https://github.com/xLLM-AI/xllm/blob/main/examples/generate.py) + +LLM Beam Search example: [:simple-github: https://github.com/xLLM-AI/xllm/blob/main/examples/generate_beam_search.py](https://github.com/xLLM-AI/xllm/blob/main/examples/generate_beam_search.py) + +Use `BeamSearchParams` with `beam_width` greater than `1`, then call `llm.beam_search(...)`: + +```python +from xllm import BeamSearchParams, LLM + +llm = LLM(model="/path/models/Qwen2-7B-Instruct", devices="npu:0") +params = BeamSearchParams( + beam_width=2, + top_logprobs=4, + max_tokens=20, +) + +outputs = llm.beam_search( + [{"prompt": "Hello, my name is "}], + params=params, +) +print(outputs[0].sequences[0].text) + +llm.finish() +``` + +For LLM Beam Search, use `beam_width` as the switch. `top_logprobs` controls the top-k candidate count used for beam expansion at each decode step. If `top_logprobs` is left at its default value, xLLM uses `beam_width` as the top logprob count. Set `top_logprobs` to a value greater than `beam_width` when you want each beam to consider more candidate tokens. This beam-search top-k is different from the sampling cutoff parameter `top_k`. `best_of` is not the Beam Search switch, and this offline LLM guide does not use `num_return_sequences` to control the returned beams. + +## Embedding + +Generate embedding example: [:simple-github: https://github.com/xLLM-AI/xllm/blob/main/examples/generate_embedding.py](https://github.com/xLLM-AI/xllm/blob/main/examples/generate_embedding.py) + +## VLM + +VLM inference example: [:simple-github: https://github.com/xLLM-AI/xllm/blob/main/examples/generate_vlm.py](https://github.com/xLLM-AI/xllm/blob/main/examples/generate_vlm.py) diff --git a/docs/src/content/docs/en/getting_started/online_service.md b/docs/src/content/docs/en/getting_started/online_service.md new file mode 100644 index 0000000000..b92bf45f42 --- /dev/null +++ b/docs/src/content/docs/en/getting_started/online_service.md @@ -0,0 +1,266 @@ +--- +title: "Online Service" +sidebar: + order: 6 +--- +First, start the xllm service according to the [xllm launch documentation](/en/getting_started/launch_xllm/). Below are examples of client calls for LLM and VLM. Please modify the parameters according to your actual situation. + +## LLM Client Calls +### HTTP Call + +Chat mode: +```bash +curl http://localhost:9977/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen2-7B-Instruct", + "max_tokens": 10, + "temperature": 0, + "stream": true, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "hello xllm" + } + ] + }' +``` + +Completions mode: +```bash +curl http://127.0.0.1:9977/v1/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen2-7B-Instruct", + "prompt": "hello xllm", + "max_tokens": 10, + "temperature": 0, + "stream": true + }' +``` + +Beam Search: + +Set `beam_width` to a value greater than `1` to enable LLM Beam Search. This parameter is supported by both `/v1/chat/completions` and `/v1/completions`. The beam-search top-k candidate count is configured with `top_logprobs` in chat requests and with the numeric `logprobs` field in completion requests. When these fields are omitted, xLLM uses `beam_width` as the top logprob count. Set the candidate count to a value greater than `beam_width` when you want each beam to consider more candidate tokens. This is different from the sampling cutoff parameter `top_k`. `best_of` is not the Beam Search switch, and this LLM API guide does not use `num_return_sequences` to control the returned beams. + +Chat mode: +```bash +curl http://localhost:9977/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen2-7B-Instruct", + "max_tokens": 20, + "temperature": 0, + "stream": false, + "beam_width": 2, + "logprobs": true, + "top_logprobs": 4, + "messages": [ + { + "role": "user", + "content": "Write a short introduction to xLLM." + } + ] + }' +``` + +Completions mode: +```bash +curl http://127.0.0.1:9977/v1/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen2-7B-Instruct", + "prompt": "Write a short introduction to xLLM.", + "max_tokens": 20, + "temperature": 0, + "stream": false, + "beam_width": 2, + "logprobs": 4 + }' +``` + +Sample mode: +```bash +curl http://127.0.0.1:9977/v1/sample \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen2-7B-Instruct", + "prompt": "Question: matched or not. Conclusion: ", + "selector": { + "type": "literal", + "value": "" + }, + "logprobs": 5, + "request_id": "sample-demo-001" + }' +``` + +Typical response: +```json +{ + "id": "sample-demo-001", + "object": "sample_completion", + "created": 1773369600, + "model": "Qwen2-7B-Instruct", + "choices": [ + { + "index": 0, + "text": "True", + "logprobs": { + "tokens": ["True", "False"], + "token_ids": [3456, 7890], + "token_logprobs": [-0.12, -2.31] + }, + "finish_reason": "selector_match" + }, + { + "index": 1, + "text": "", + "logprobs": { + "tokens": [], + "token_ids": [], + "token_logprobs": [] + }, + "finish_reason": "empty_logprobs" + } + ], + "usage": { + "prompt_tokens": 20, + "completion_tokens": 2, + "total_tokens": 22 + } +} +``` + +`/v1/sample` notes: + +- Only `--backend=llm` is supported. VLM/DiT/Rec are not supported yet. +- `selector.type` is currently fixed to `literal`. `selector.value` is matched against prompt text in full and in order. +- `logprobs` defaults to `5`, with an allowed range of `[1, 5]`. +- `choices[i].index` is the matched `sample_id`, corresponding one-to-one with the matched order in prompt. +- If no selector match is found, the service returns `200` with `choices=[]`. If a matched position has no available logprobs, it returns `finish_reason="empty_logprobs"`. +- Service logs only summary fields such as `request_id`, `sample_id`, `match_count`, and `model`, and do not log the full prompt. + +`/v1/sample` common error semantics: + +- Missing `model/prompt/selector/selector.value`, `selector.type != literal`, or out-of-range `logprobs` returns `INVALID_ARGUMENT`. +- If the model does not exist or the backend is not `llm`, it returns `UNKNOWN`. +- When concurrency reaches the upper limit, it returns `RESOURCE_EXHAUSTED`. +- When the model is in sleep state, it returns `UNAVAILABLE`. + +### Python Call +```python +import requests +import json + +url = f"http://localhost:9977/v1/chat/completions" +messages = [ + {'role': 'user', 'content': "List three countries and their capitals."} +] + +request_data = { + "model": "Qwen2-7B-Instruct", + "messages": messages, + "stream": False, + "temperature": 0.6, + "max_tokens": 2048, +} + +response = requests.post(url, json=request_data) +if response.status_code != 200: + print(response.status_code, response.text) +else: + ans = json.loads(response.text)["choices"] + print(ans[0]['message']) +``` + +## VLM Client Calls +### HTTP API + +```python +import base64 +import requests + +api_url = "http://localhost:12345/v1/chat/completions" +image_url = "" + +def encode_image(url: str) -> str: + with requests.get(url) as response: + response.raise_for_status() + result = base64.b64encode(response.content).decode("utf-8") + + return result + +image_base64 = encode_image(image_url) +payload = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"}, + }, + ], + } + ], + "model": "Qwen2.5-VL-7B-Instruct", + "max_completion_tokens": 128, +} + +response = requests.post( + api_url, + json=payload, + headers={"Content-Type": "application/json"} +) +print(response.json()) +``` + +### OpenAI API +```python +from openai import OpenAI +import base64 +import requests + +openai_api_key = "EMPTY" +openai_api_base = "http://localhost:12345/v1" +image_url = "" + +client = OpenAI( + api_key=openai_api_key, + base_url=openai_api_base, +) + +def encode_image(url: str) -> str: + with requests.get(url) as response: + response.raise_for_status() + result = base64.b64encode(response.content).decode("utf-8") + + return result + +image_base64 = encode_image(image_url) +chat_completion = client.chat.completions.create( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"}, + }, + ], + } + ], + model="Qwen2.5-VL-7B-Instruct", + max_completion_tokens=128, +) + +result = chat_completion.choices[0].message.content +print("Chat completion output:", result) +``` diff --git a/docs/src/content/docs/en/getting_started/quick_start.md b/docs/src/content/docs/en/getting_started/quick_start.md new file mode 100644 index 0000000000..cd1d534bd8 --- /dev/null +++ b/docs/src/content/docs/en/getting_started/quick_start.md @@ -0,0 +1,201 @@ +--- +title: "Quick Start" +sidebar: + order: 1 +--- +## Environment Setup + +All images are stored [here](https://quay.io/repository/jd_xllm/xllm-ai?tab=tags). The docker startup command below uses the dev image as an example. + +### NPU + +Below are our pre-built dev image. +```bash +# A2 x86 +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-x86-20260306 +# A2 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-arm-20260306 +# A3 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-20260306 +``` + +Container startup command: +```bash +docker run -it \ +--ipc=host \ +-u 0 \ +--name xllm-npu \ +--privileged \ +--network=host \ +--device=/dev/davinci0 \ +--device=/dev/davinci_manager \ +--device=/dev/devmm_svm \ +--device=/dev/hisi_hdc \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ +-v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \ +-v /usr/local/sbin/:/usr/local/sbin/ \ +-v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \ +-v /var/log/npu/slog/:/var/log/npu/slog \ +-v /var/log/npu/profiling/:/var/log/npu/profiling \ +-v /var/log/npu/dump/:/var/log/npu/dump \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +### NVIDIA GPU + +We provide a [Dockerfile](https://github.com/xLLM-AI/xllm/blob/main/docker/Dockerfile.cuda) for NVIDIA GPU usage, which can be used to build custom image. Of course, you can also use dev image we built based on the default Dockerfile: +```bash +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-cuda-x86 +``` + +Container startup command: +```bash +sudo docker run -it \ +--privileged \ +--shm-size '128gb' \ +--ipc=host \ +--net=host \ +--pid=host \ +--name=xllm-cuda \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +### MLU + +We cannot provide MLU image. If you already have the dev image, you can start the container with the following command: +```bash +sudo docker run -it \ +--privileged \ +--shm-size '128gb' \ +--ipc=host \ +--net=host \ +--pid=host \ +--name xllm-mlu \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +### Hygon DCU + +Below are our pre-built dev image. +```bash +docker pull harbor.sourcefind.cn:5443/dcu/admin/base/custom:xllm-dev-dcu-x86-20260617 +``` + +Container startup command: +```bash +docker run -it \ +--ipc=host \ +-u 0 \ +--name xllm-dcu \ +--privileged \ +--network=host \ +--shm-size 256g \ +--device=/dev/kfd \ +--device=/dev/dri \ +--device=/dev/mkfd \ +--security-opt seccomp=unconfined \ +--group-add video \ +-v /opt/hyhal:/opt/hyhal \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +### MetaX MACA + +Below are our pre-built dev image. +```bash +docker pull pub-registry1.metax-tech.com/dev-m01421/xllm-maca3.7.1.9:v1 +``` + +Container startup command: +```bash +docker run -it \ +--ipc=host \ +-u 0 \ +--name xllm-maca \ +--network=host \ +--privileged=true \ +--shm-size 100gb \ +--device=/dev/mxcd \ +--device=/dev/dri \ +--device=/dev/infiniband \ +--security-opt seccomp=unconfined \ +--security-opt apparmor=unconfined \ +--group-add video \ +--ulimit memlock=-1 \ +-v /opt/maca:/opt/maca \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +### Mthreads MUSA + +Image pull: + +```bash +docker pull registry.mthreads.com/presale/devtech/xllm:0710 +``` + +Container startup: + +```bash +docker run -it \ + --ipc=host \ + --network=host \ + --privileged \ + --shm-size=128g \ + --name xllm-musa \ + --device=/dev/mtgpu0 \ + --device=/dev/dri \ + --group-add video \ + --ulimit memlock=-1 \ + -v $HOME:$HOME \ + -w $HOME \ + registry.mthreads.com/presale/devtech/xllm:0710 \ + /bin/bash +``` + +See [Mthreads MUSA](/en/hardware/musa/) for full details. + +## Build xllm + +If you download a release image, i.e., an image with a version number in the tag, you can skip this step because the release image comes with a pre-compiled xllm binary, and call `xllm` directly. + +Download xllm and dependencies: +```bash +git clone https://github.com/xLLM-AI/xllm.git +cd xllm + +# Install pre-commit for the first time +pip install pre-commit +pre-commit install + +git submodule update --init --recursive +``` + +In a new image, the first compilation of xllm takes a long time because all dependencies in vcpkg need to be compiled, but subsequent compilations will be much faster. +```bash +# Compile cpp binary +python setup.py build + +# Compile python wheel +python setup.py bdist_wheel +``` + +## Launch xllm +Please refer to [How to Launch xllm](/en/getting_started/launch_xllm/). + diff --git a/docs/src/content/docs/en/hardware/ascend_npu.md b/docs/src/content/docs/en/hardware/ascend_npu.md new file mode 100644 index 0000000000..fbdb261286 --- /dev/null +++ b/docs/src/content/docs/en/hardware/ascend_npu.md @@ -0,0 +1,100 @@ +--- +title: "Ascend NPU" +description: "Run xLLM on Ascend NPUs with the NPU backend." +sidebar: + order: 3 +--- + +Use the NPU backend when running xLLM on Ascend hardware. + +## Image and Container Startup + +Pull a pre-built Ascend development image: + +```bash +# A2 x86 +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-x86-20260306 +# A2 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-arm-20260306 +# A3 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-20260306 +``` + +Start the container: + +```bash +docker run -it \ +--ipc=host \ +-u 0 \ +--name xllm-npu \ +--privileged \ +--network=host \ +--device=/dev/davinci0 \ +--device=/dev/davinci_manager \ +--device=/dev/devmm_svm \ +--device=/dev/hisi_hdc \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ +-v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \ +-v /usr/local/sbin/:/usr/local/sbin/ \ +-v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \ +-v /var/log/npu/slog/:/var/log/npu/slog \ +-v /var/log/npu/profiling/:/var/log/npu/profiling \ +-v /var/log/npu/dump/:/var/log/npu/dump \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +## Server Startup Command + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +export ASCEND_RT_VISIBLE_DEVICES=0 +export HCCL_IF_BASE_PORT=43432 # HCCL communication base port + +MODEL_PATH="/path/to/model/Qwen3-8B" # Model path +MASTER_NODE_ADDR="127.0.0.1:9748" # Master node address (must be globally consistent) +START_PORT=18000 # Service starting port +START_DEVICE=0 # Starting logical device number +LOG_DIR="log" # Log directory +NNODES=1 # Number of nodes (current script launches 1 process) + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="npu:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --max_memory_utilization=0.86 \ + --block_size=128 \ + --communication_backend="hccl" \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=true \ + --enable_schedule_overlap=true \ + --enable_shm=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +For distributed NPU serving, set `--communication_backend="hccl"`, keep `--master_node_addr`, `--node_rank`, and `--nnodes` consistent across all workers, and prepare the rank table when the deployment requires one. + +## Notes + +- Pre-built A2 and A3 development images are listed in [Quick Start](/en/getting_started/quick_start/). +- NPU launch examples usually enable HCCL communication and tune `--block_size`, `--max_memory_utilization`, chunked prefill, and schedule overlap for the target workload. +- Ascend kernel implementation details are documented separately in the TileLang kernel development guide. diff --git a/docs/src/content/docs/en/hardware/cambricon_mlu.md b/docs/src/content/docs/en/hardware/cambricon_mlu.md new file mode 100644 index 0000000000..981af96a70 --- /dev/null +++ b/docs/src/content/docs/en/hardware/cambricon_mlu.md @@ -0,0 +1,69 @@ +--- +title: "Cambricon MLU" +description: "Run xLLM on Cambricon MLU devices with the MLU backend." +sidebar: + order: 4 +--- + +Use the MLU backend when running xLLM on Cambricon devices. + +## Image and Container Startup + +xLLM does not currently provide a public MLU image in the docs. If you already have the development image, start the container with: + +```bash +sudo docker run -it \ +--privileged \ +--shm-size '128gb' \ +--ipc=host \ +--net=host \ +--pid=host \ +--name xllm-mlu \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +## Server Startup Command + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export MLU_VISIBLE_DEVICES=0 + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="mlu:$DEVICE" \ + --port $PORT \ + --nnodes=$NNODES \ + --master_node_addr=$MASTER_NODE_ADDR \ + --block_size=16 \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +For a single-device run, `` usually starts from `0`. For larger deployments, keep the selected device ids aligned with `--node_rank`, `--nnodes`, and per-worker ports. + +## Notes + +- xLLM does not currently provide a public MLU image in the docs. Use an available MLU development image with the container startup command above. +- The MLU launch example uses `--block_size=16` in the current docs. +- Check the [Model Support List](/en/supported_models/) before choosing an MLU deployment target. diff --git a/docs/src/content/docs/en/hardware/dcu.md b/docs/src/content/docs/en/hardware/dcu.md new file mode 100644 index 0000000000..33ec7a6f6c --- /dev/null +++ b/docs/src/content/docs/en/hardware/dcu.md @@ -0,0 +1,81 @@ +--- +title: "Hygon DCU" +description: "Run xLLM on Hygon DCU devices with the Hygon DCU backend." +sidebar: + order: 5 +--- + +Use the Hygon DCU backend when running xLLM on Hygon DCU hardware. + +## Image and Container Startup + +Pull the Hygon DCU development image: + +```bash +docker pull harbor.sourcefind.cn:5443/dcu/admin/base/custom:xllm-dev-dcu-x86-20260617 +``` + +Start the container: + +```bash +docker run -it \ +--ipc=host \ +-u 0 \ +--name xllm-dcu \ +--privileged \ +--network=host \ +--shm-size 256g \ +--device=/dev/kfd \ +--device=/dev/dri \ +--device=/dev/mkfd \ +--security-opt seccomp=unconfined \ +--group-add video \ +-v /opt/hyhal:/opt/hyhal \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +## Server Startup Command + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export HIP_VISIBLE_DEVICES=0 + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="dcu:$DEVICE" \ + --port $PORT \ + --nnodes=$NNODES \ + --master_node_addr=$MASTER_NODE_ADDR \ + --block_size=128 \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +For a single-device run, `` usually starts from `0`. For multi-worker deployments, keep device ids, `--node_rank`, `--nnodes`, and service ports aligned. + +## Notes + +- The current docs list a pre-built Hygon DCU development image in [Quick Start](/en/getting_started/quick_start/). +- The Hygon DCU container startup command requires device mounts such as `/dev/kfd`, `/dev/dri`, and `/dev/mkfd`; the command above includes these mounts. +- Check the [Model Support List](/en/supported_models/) before choosing a Hygon DCU deployment target. diff --git a/docs/src/content/docs/en/hardware/metax_maca.md b/docs/src/content/docs/en/hardware/metax_maca.md new file mode 100644 index 0000000000..073e60c804 --- /dev/null +++ b/docs/src/content/docs/en/hardware/metax_maca.md @@ -0,0 +1,88 @@ +--- +title: "MetaX MACA" +description: "Run xLLM on MetaX MACA devices with the MetaX MACA backend." +sidebar: + order: 6 +--- + +Use the MetaX MACA backend when running xLLM on MetaX MACA hardware. + +## Image and Container Startup + +Pull the MetaX MACA development image: + +```bash +docker pull pub-registry1.metax-tech.com/dev-m01421/xllm-maca3.7.1.9:v1 +``` + +Start the container: + +```bash +docker run -it \ +--ipc=host \ +-u 0 \ +--name xllm-maca \ +--network=host \ +--privileged=true \ +--shm-size 100gb \ +--device=/dev/mxcd \ +--device=/dev/dri \ +--device=/dev/infiniband \ +--security-opt seccomp=unconfined \ +--security-opt apparmor=unconfined \ +--group-add video \ +--ulimit memlock=-1 \ +-v /opt/maca:/opt/maca \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +## Server Startup Command + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export CUDA_VISIBLE_DEVICES=0 +export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/data/aot/ + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="cuda:$DEVICE" \ + --port $PORT \ + --nnodes=$NNODES \ + --master_node_addr=$MASTER_NODE_ADDR \ + --block_size=128 \ + --max_memory_utilization=0.86 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_schedule_overlap=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +For a single-device run, `` usually starts from `0`. For multi-worker deployments, keep device ids, `--node_rank`, `--nnodes`, and service ports aligned. + +## Notes + +- The current docs list a pre-built MetaX MACA development image in [Quick Start](/en/getting_started/quick_start/). +- The MetaX MACA container startup command requires device mounts such as `/dev/mxcd`, `/dev/dri`, and `/dev/infiniband`; the command above includes these mounts. +- Build xllm with MetaX MACA: python setup.py build --device maca diff --git a/docs/src/content/docs/en/hardware/musa.md b/docs/src/content/docs/en/hardware/musa.md new file mode 100644 index 0000000000..5471398d08 --- /dev/null +++ b/docs/src/content/docs/en/hardware/musa.md @@ -0,0 +1,84 @@ +--- +title: "Mthreads MUSA" +description: "Run xLLM on Mthreads MUSA GPUs." +sidebar: + order: 7 +--- + +Use the MUSA device backend (`--devices=musa:`) when running xLLM on Mthreads MUSA hardware. + +## Image and Container Startup + +Pull the Mthreads xLLM image: + +```bash +docker pull registry.mthreads.com/presale/devtech/xllm:0710 +``` + +Start the container: + +```bash +docker run -it \ + --ipc=host \ + --network=host \ + --privileged \ + --shm-size=128g \ + --name xllm-musa \ + --device=/dev/mtgpu0 \ + --device=/dev/dri \ + --group-add video \ + --ulimit memlock=-1 \ + -v $HOME:$HOME \ + -w $HOME \ + registry.mthreads.com/presale/devtech/xllm:0710 \ + /bin/bash +``` + +Add `--device=/dev/mtgpuN` for additional GPUs. Select the physical GPU with `export MUSA_VISIBLE_DEVICES=0`. + +## Server Startup Command + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export MUSA_VISIBLE_DEVICES=0 + +MODEL_PATH="/path/to/model/Qwen3.5-27B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="musa:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --block_size=64 \ + --max_memory_utilization=0.8 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=true \ + --enable_schedule_overlap=true \ + --enable_graph=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +For a single-device run, the logical device id usually starts at `0`. For multi-GPU or multi-node deployments, keep device ids, `--node_rank`, `--nnodes`, and service ports aligned with your topology. + +## Notes + +- Image and container startup are also listed in [Quick Start](/en/getting_started/quick_start/). +- Launch flags and multi-node setup are covered in [Launch xllm](/en/getting_started/launch_xllm/). diff --git a/docs/src/content/docs/en/hardware/nvidia_gpu.md b/docs/src/content/docs/en/hardware/nvidia_gpu.md new file mode 100644 index 0000000000..be797ce95d --- /dev/null +++ b/docs/src/content/docs/en/hardware/nvidia_gpu.md @@ -0,0 +1,83 @@ +--- +title: "NVIDIA GPU" +description: "Run xLLM on NVIDIA GPUs with the CUDA backend." +sidebar: + order: 2 +--- + +Use the CUDA backend when running xLLM on NVIDIA GPUs. + +## Image and Container Startup + +Pull the CUDA development image: + +```bash +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-cuda-x86 +``` + +Start the container: + +```bash +sudo docker run -it \ +--privileged \ +--shm-size '128gb' \ +--ipc=host \ +--net=host \ +--pid=host \ +--name=xllm-cuda \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +## Server Startup Command + +Set visible CUDA devices before launching the service: + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export CUDA_VISIBLE_DEVICES=0 +# for debug +# export CUDA_LAUNCH_BLOCKING=1 + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="cuda:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --block_size=32 \ + --max_memory_utilization=0.8 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_schedule_overlap=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +For a single-device run, `` usually starts from `0`. For multi-device or multi-node runs, keep the device ids, `--node_rank`, `--nnodes`, and service ports consistent with the launch topology. + +## Notes + +- The CUDA development image and Dockerfile references are maintained in [Quick Start](/en/getting_started/quick_start/). +- The default CUDA launch example uses `--block_size=32` and `--max_memory_utilization=0.8`; adjust these values according to the target model and available GPU memory. +- CUDA timeline collection is available through the profiling endpoints described in [Online Profiling](/en/dev_guide/online_profiling/). diff --git a/docs/src/content/docs/en/hardware/overview.md b/docs/src/content/docs/en/hardware/overview.md new file mode 100644 index 0000000000..417011e7c2 --- /dev/null +++ b/docs/src/content/docs/en/hardware/overview.md @@ -0,0 +1,24 @@ +--- +title: "Hardware Platforms" +description: "Platform-specific guides for running xLLM on supported hardware." +sidebar: + order: 1 +--- + +xLLM supports multiple accelerator backends for large-scale model inference. This section collects the hardware-specific entry points for environment setup, runtime device selection, launch scripts, and model support. + +## Platform Guides + +- [NVIDIA GPU](/en/hardware/nvidia_gpu/) - CUDA backend setup and launch entry points. +- [Ascend NPU](/en/hardware/ascend_npu/) - Ascend NPU setup, runtime environment, and HCCL launch notes. +- [Cambricon MLU](/en/hardware/cambricon_mlu/) - MLU backend setup and launch entry points. +- [Hygon DCU](/en/hardware/dcu/) - Hygon DCU backend setup and launch entry points. +- [MetaX MACA](/en/hardware/metax_maca/) - MetaX MACA backend setup and launch entry points. +- [Mthreads MUSA](/en/hardware/musa/) - Mthreads MUSA GPU image, build, and `--devices=musa` launch entry points. + +## Common Workflow + +1. Prepare the platform-specific container image from the explicit commands in each platform guide. +2. Build xLLM inside the container, or use a release image that already includes `xllm`. +3. Start the service with the matching device backend in [Launch xllm](/en/getting_started/launch_xllm/). +4. Check model and modality coverage in the [Model Support List](/en/supported_models/). diff --git a/docs/src/content/docs/en/index.md b/docs/src/content/docs/en/index.md new file mode 100644 index 0000000000..9c2c1df518 --- /dev/null +++ b/docs/src/content/docs/en/index.md @@ -0,0 +1,49 @@ +--- +title: "xLLM" +description: "xLLM documentation home" +head: + - tag: title + content: "xLLM Documentation" +--- +![xLLM](../assets/logo_with_llm.png) + +## Project Overview + +**xLLM** is an efficient and user-friendly LLM intelligent inference framework that provides enterprise-level service guarantees and high-performance engine computing capabilities for model inference on domestic AI accelerators. + +#### Background + +LLM with parameter scales ranging from tens of billions to trillions are being rapidly deployed in core business scenarios such as intelligent customer service, real-time recommendation, and content generation. Efficient support for domestic computing hardware has become a core requirement for low-cost inference deployment. Existing inference engines struggle to effectively adapt to the architectural characteristics of dedicated accelerators like domestic chips. Performance issues such as low utilization of computing units, load imbalance and communication overhead bottlenecks under the MoE architecture, and difficulties in kv cache management have restricted the efficient inference of requests and the scalability of the system. The xLLM inference engine improves the resource efficiency of the entire "communication-computation-storage" performance link and it provides crucial technical support for the large-scale implementation of LLM in real-world business scenarios. + +## Core Features + +**xLLM** delivers robust intelligent computing capabilities. By leveraging hardware system optimization and algorithm-driven decision control, it jointly accelerates the inference process, enabling high-throughput, low-latency distributed inference services. + +**Full Graph Pipeline Execution Orchestration** +- Asynchronous decoupled scheduling at the requests scheduling layer, to reduce computational bubbles. +- Asynchronous parallelism of computation and communication at the model graph layer, overlapping computation and communication. +- Pipelining of heterogeneous computing units at the operator kernel layer, overlapping computation and memory access. + +**Graph Optimization for Dynamic Shapes** +- Dynamic shape adaptation based on parameterization and multi-graph caching methods to enhance the flexibility of static graph. +- Controlled tensor memory pool to ensure address security and reusability. +- Integration and adaptation of performance-critical custom operators (e.g., *PageAttention*, *AllReduce*). + +**MoE Kernel Optimization** +- *GroupMatmul* optimization to improve computational efficiency. +- Chunked Prefill optimization to support long-sequence inputs. + +**Efficient Memory Optimization** +- Mapping management between discrete physical memory and continuous virtual memory. +- On-demand memory allocation to reduce memory fragmentation. +- Intelligent scheduling of memory pages to increase memory reusability. +- Adaptation of corresponding operators for domestic accelerators. + +**Global KV Cache Management** +- Intelligent offloading and prefetching of KV in hierarchical caches. +- KV cache-centric distributed storage architecture. +- Intelligent KV routing among computing nodes. + +**Algorithm-driven Acceleration** +- Speculative decoding optimization to improve efficiency through multi-core parallelism. +- Dynamic load balancing of MoE experts to achieve efficient adjustment of expert distribution. diff --git a/docs/src/content/docs/en/supported_models.md b/docs/src/content/docs/en/supported_models.md new file mode 100644 index 0000000000..f8a2bbb2f1 --- /dev/null +++ b/docs/src/content/docs/en/supported_models.md @@ -0,0 +1,49 @@ +--- +title: "Model Support List" +sidebar: + order: 20 +--- +## LLM +| | NPU | MLU | ILU | Hygon DCU | +| ------------------------ | :---: | :---: | :---: | :---: | +| DeepSeek-V3/R1/V3.1 | ✅ | ✅ | ❌ | ❌ | +| DeepSeek-V3.2 | ✅ | ✅ | ❌ | ❌ | +| DeepSeek-R1-Distill-Qwen | ✅ | ❌ | ❌ | ❌ | +| Qwen2/2.5/QwQ | ✅ | ✅ | ✅ | ✅ | +| Qwen3 | ✅ | ✅ | ✅ | ✅ | +| Qwen3 Moe | ✅ | ✅ | ✅ | ✅ | +| Kimi-k2 | ✅ | ❌ | ❌ | ❌ | +| Llama2/3 | ✅ | ❌ | ✅ | ❌ | +| GLM4.5 | ✅ | ❌ | ❌ | ❌ | +| GLM4.6 | ✅ | ❌ | ❌ | ❌ | +| GLM-4.7 | ✅ | ❌ | ❌ | ❌ | +| GLM-5 | ✅ | ❌ | ❌ | ❌ | + +## VLM +| | NPU | MLU | ILU | Hygon DCU | +| ------------ | :---: | :---: | :---: | :---: | +| MiniCPM-V | ✅ | ❌ | ❌ | ❌ | +| MiMo-VL | ✅ | ❌ | ❌ | ❌ | +| Qwen2.5-VL | ✅ | ✅ | ❌ | ✅ | +| Qwen3-VL | ✅ | ✅ | ❌ | ✅ | +| Qwen3-VL-MoE | ✅ | ✅ | ❌ | ✅ | +| GLM-4.6V | ✅ | ❌ | ❌ | ❌ | +| VLM-R1 | ✅ | ❌ | ❌ | ❌ | + +## Rerank +| | NPU | MLU | ILU | Hygon DCU | +| -------------- | :---: | :---: | :---: | :---: | +| Qwen3-Reranker | ✅ | ❌ | ❌ | ❌ | + +## DiT +| | NPU | MLU | ILU | Hygon DCU | +| ---- | :---: | :---: | :---: | :---: | +| Flux | ✅ | ❌ | ❌ | ❌ | + +## Rec +| | NPU | MLU | ILU | Hygon DCU | +| --- | :---: | :---: | :---: | :---: | +| OneRec | ✅ | ❌ | ❌ | ❌ | +| Qwen2 | ✅ | ❌ | ❌ | ❌ | +| Qwen2.5 | ✅ | ❌ | ❌ | ❌ | +| Qwen3 | ✅ | ❌ | ❌ | ❌ | diff --git a/docs/src/content/docs/zh/cli_reference.md b/docs/src/content/docs/zh/cli_reference.md new file mode 100644 index 0000000000..425f668f19 --- /dev/null +++ b/docs/src/content/docs/zh/cli_reference.md @@ -0,0 +1,285 @@ +--- +title: "服务启动参数" +sidebar: + order: 100 +--- + +xLLM 使用 gflags 管理服务启动参数。`--model ` 是唯一必填参数。使用 `--config_json_file` 时，JSON 文件中的值会覆盖命令行 flag 值。下表按 `/xllm/core/framework/config` 下的 Config 类分组，一个 Config 对应一节；`ConfigJsonUtils` 一节包含配置文件相关的通用参数。 + +## ConfigJsonUtils + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `config_json_file` | `string` | `""` | JSON 配置文件路径；文件中的值会覆盖命令行 flag 值。 | +| `enable_dump_config_json` | `bool` | `false` | 是否将最终生效的启动配置导出为 JSON。 | +| `dump_config_json_file` | `string` | `"xllm_config.json"` | 导出启动配置 JSON 的路径，仅在 `enable_dump_config_json=true` 时使用。 | + +## ServiceConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `host` | `string` | `""` | bRPC 服务监听的主机名或 IP。 | +| `port` | `int32` | `8010` | bRPC 服务监听端口。 | +| `rpc_idle_timeout_s` | `int32` | `-1` | 连接在最近 `rpc_idle_timeout_s` 秒内无读写操作时关闭；`-1` 表示无限等待。 | +| `rpc_channel_timeout_ms` | `int32` | `-1` | bRPC Channel 的最大等待时间，单位毫秒；`-1` 表示无限等待。 | +| `max_reconnect_count` | `int32` | `40` | worker 尝试连接 server 的最大重连次数。 | +| `num_threads` | `int32` | `8` | 处理请求的线程数。 | +| `max_concurrent_requests` | `int32` | `200` | 实例可同时处理的最大请求数；设为 `0` 表示不限流。 | +| `num_request_handling_threads` | `int32` | `4` | 处理输入请求的线程数。 | +| `num_response_handling_threads` | `int32` | `4` | 处理响应输出的线程数。 | +| `health_check_interval_ms` | `int32` | `3000` | worker 健康检查间隔，单位毫秒。 | + +## ModelConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `model_id` | `string` | `""` | Hugging Face 模型名称，非路径。 | +| `model` | `string` | `""` | Hugging Face 模型名称或模型路径。 | +| `backend` | `string` | `""` | 后端模型类型；`llm` 表示纯文本模型，`vlm` 表示多模态模型，`dit` 表示扩散模型。 | +| `task` | `string` | `"generate"` | 模型任务类型，例如 `generate`、`embed`、`mm_embed`。 | +| `devices` | `string` | `""` | 已废弃，请改用 `device_id`。当前进程使用的设备，例如 `npu:0`、`npu:0,npu:1`。 | +| `device_id` | `int32` | `-1` | 运行模型的 device id，例如 `0`。 | +| `limit_image_per_prompt` | `int32` | `8` | 每个 prompt 允许的最大图片数量，仅用于多模态模型。 | +| `max_encoder_cache_size` | `int64` | `0` | 每个 worker 的 encoder cache 最大显存大小，单位 MB；`0` 表示禁用 encoder cache。 | +| `reasoning_parser` | `string` | `""` | reasoning 交互解析器，例如 `auto`、`glm45`、`glm47`、`glm5`、`qwen3`、`qwen35`、`deepseek-r1`。 | +| `tool_call_parser` | `string` | `""` | tool-call 交互解析器，例如 `auto`、`qwen25`、`qwen3`、`qwen35`、`qwen3_coder`、`kimi_k2`、`deepseekv3`、`deepseekv32`、`deepseekv4`、`glm45`、`glm47`、`glm5`。 | +| `enable_qwen3_reranker` | `bool` | `false` | 是否启用 Qwen3 reranker。 | +| `flashinfer_workspace_buffer_size` | `int32` | `134217728` | FlashInfer split-k attention 中用于保存中间 attention 结果的预留 workspace 大小，默认 128 MiB。 | +| `enable_return_mm_full_embeddings` | `bool` | `false` | VLM 模型是否返回 ViT embedding 与序列 embedding。 | +| `mm_download_headers` | `string` | `""` | 多模态下载的 service 级默认 HTTP header，为 JSON 对象；per-request header 优先级更高。示例：`{"Authorization":"Bearer xxx"}`。 | +| `use_audio_in_video` | `bool` | `false` | 输入为视频时，是否同时解码音频和视频。 | +| `use_cpp_chat_template` | `bool` | `true` | 对支持的模型使用原生 C++ chat template，例如 `deepseek_v32`、`deepseek_v4`；设为 `false` 可回退到 Jinja 以便调试。 | + +## LoadConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `enable_manual_loader` | `bool` | `false` | 将 decoder layer 权重固定在 host 内存并使用异步 H2D 传输；`enable_rolling_load` 依赖该参数，`enable_xtensor` 也会隐式启用该能力。 | +| `enable_rolling_load` | `bool` | `false` | 启用 rolling weight load：HBM 中仅保留 N 个 decoder layer 权重槽位，并按层即时加载；需要 `enable_manual_loader=true`，仅 NPU 支持。 | +| `rolling_load_num_cached_layers` | `int32` | `2` | `enable_rolling_load=true` 时 HBM 中保留的 decoder layer 权重槽位数量。 | +| `rolling_load_num_rolling_slots` | `int32` | `-1` | decoder rolling load 使用的 rolling 槽位数量；固定槽位数为 `rolling_load_num_cached_layers - rolling_load_num_rolling_slots`。`-1` 表示自动设置为 `min(2, preload_count)`，取值需在 `[-1, rolling_load_num_cached_layers]` 范围内。 | +| `enable_prefetch_weight` | `bool` | `false` | 是否启用权重预取，仅适用于 Qwen3-dense 模型；gateup 权重默认预取比例为 40%，可通过环境变量 `PREFETCH_COEFFOCIENT` 调整。 | + +## KVCacheConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `block_size` | `int32` | `128` | 每个 KV Cache block 的 slot 数。 | +| `max_cache_size` | `int64` | `0` | KV Cache 可使用的 GPU 显存大小；`0` 表示根据可用显存自动计算。 | +| `max_memory_utilization` | `double` | `0.8` | 模型推理可使用的 GPU 显存比例，包括模型权重和 KV Cache。 | +| `kv_cache_dtype` | `string` | `"auto"` | KV Cache 量化数据类型；`auto` 表示与模型 dtype 对齐且不量化，`int8` 表示启用 INT8 量化，仅 MLU 后端支持。 | +| `enable_prefix_cache` | `bool` | `true` | 是否在 block manager 中启用 prefix cache；详见 [Prefix Cache](/zh/features/prefix_cache/)。 | +| `enable_in_batch_prefix_cache` | `bool` | `false` | 是否将已准入的 prefill 完整 block 缓存进 prefix cache，使同一 batch 内的后续请求可以共享。 | +| `max_linear_state_cache_slots` | `int64` | `0` | linear-attention state cache 的最大活跃槽位数；`0` 表示根据可用 KV Cache 预算自动推导容量。 | +| `xxh3_128bits_seed` | `uint32` | `1024` | XXH3 128-bit 哈希的默认 seed。 | +| `enable_xtensor` | `bool` | `false` | 是否为模型权重启用基于物理页池的 XTensor。 | +| `phy_page_granularity_size` | `int64` | `2097152` | 单个物理页的粒度大小，单位 byte，默认 2 MiB；用于连续 KV Cache。 | + +## KVCacheStoreConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `prefetch_timeout` | `uint32` | `0` | 从 KV Cache Store 预取数据的超时时间。 | +| `prefetch_batch_size` | `uint32` | `2` | 从 KV Cache Store 预取并拷贝的 batch size。 | +| `layers_wise_copy_batchs` | `uint32` | `4` | 按层执行 H2D 拷贝的 batch 数。 | +| `host_blocks_factor` | `double` | `0.0` | host block 系数，例如 `host block num = host_blocks_factor * hbm block num`。 | +| `enable_kvcache_store` | `bool` | `false` | 是否启用 KV Cache Store。 | +| `store_protocol` | `string` | `"tcp"` | KV Cache Store 协议，例如 `tcp`、`rdma`。 | +| `store_master_server_address` | `string` | `""` | Store master service 的地址信息。 | +| `store_metadata_server` | `string` | `""` | KV Cache Store metadata service 的地址。 | +| `store_local_hostname` | `string` | `""` | KV Cache Store client 的本地主机名。 | +| `enable_control_h2d_block_num` | `bool` | `false` | 是否控制 H2D 拷贝的 block 数。 | + +## BeamSearchConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `enable_beam_search_kernel` | `bool` | `false` | 是否启用 beam search kernel。 | +| `beam_width` | `int32` | `1` | Beam search 的 beam width。 | +| `enable_block_copy_kernel` | `bool` | `true`（NPU/CUDA）；`false`（其他后端） | 是否在支持的后端使用 block copy kernel。 | +| `enable_topk_sorted` | `bool` | `true` | 是否启用 top-k 结果排序输出。 | + +## SchedulerConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `max_tokens_per_batch` | `int32` | `10240` | 每个 batch 可处理的最大 token 数。 | +| `max_seqs_per_batch` | `int32` | `1024` | 每个 batch 可处理的最大 sequence 数。 | +| `enable_schedule_overlap` | `bool` | `false` | 是否启用 schedule overlap（异步调度）；详见 [异步调度](/zh/features/async_schedule/)。 | +| `prefill_scheduling_memory_usage_threshold` | `double` | `0.95` | prefill 调度时的内存使用阈值。 | +| `enable_chunked_prefill` | `bool` | `true` | 是否启用 chunked prefill。 | +| `max_tokens_per_chunk_for_prefill` | `int32` | `-1` | prefill 阶段每个 chunk 的最大 token 数；`-1` 表示使用默认策略。 | +| `chunked_match_frequency` | `int32` | `2` | sequence prefix cache 匹配频率。 | +| `use_zero_evict` | `bool` | `false` | 是否使用 ZeroEvictionScheduler；详见 [Zero Evict Scheduler](/zh/features/zero_evict_scheduler/)。 | +| `max_decode_token_per_sequence` | `int32` | `256` | ZeroEvictionScheduler 中每个 sequence 的最大 decode token 数。 | +| `priority_strategy` | `string` | `"fcfs"` | 请求优先级策略，例如 `fcfs`、`priority`、`deadline`。 | +| `use_mix_scheduler` | `bool` | `false` | 是否使用 MixScheduler 统一处理 prefill 和 decode。 | +| `enable_online_preempt_offline` | `bool` | `true` | 是否允许在线请求抢占离线请求。 | +| `aggressive_coeff` | `double` | `1.0` | MixScheduler 紧急度判断的激进系数。 | +| `starve_threshold` | `double` | `1.0` | MixScheduler 的饥饿阈值系数。 | +| `enable_starve_prevent` | `bool` | `true` | 是否启用 MixScheduler 的防饥饿机制。 | + +## ParallelConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `dp_size` | `int32` | `1` | MLA attention 的 data parallel 规模。 | +| `ep_size` | `int32` | `1` | MoE 模型的 expert parallel 规模。 | +| `cp_size` | `int32` | `1` | DSA attention 的 context parallel 规模。 | +| `kv_split_size` | `int32` | `1` | KV Cache split 宽度；`0` 表示沿用 `cp_size`（旧行为），`1` 表示不切分 KV（每个 CP rank 存储完整 KV 并跳过 prefix AllGather），其他可整除 `cp_size` 的 K 表示 KV 在 K 个 rank 间分片，而 token-CP 仍使用 `cp_size`。 | +| `tp_size` | `int64` | `1` | Tensor parallel 规模，仅 DiT 模型使用。 | +| `sp_size` | `int64` | `1` | Sequence parallel 规模，仅 DiT 模型使用。 | +| `cfg_size` | `int64` | `1` | Classifier-free guidance parallel 规模，仅 DiT 模型使用。 | +| `vae_size` | `int64` | `1` | VAE patch parallel 规模，仅 DiT 模型使用。 | +| `communication_backend` | `string` | `"hccl"` | NPU 通信后端，例如 `lccl`、`hccl`；启用 DP 时使用 `hccl`。 | +| `enable_prefill_sp` | `bool` | `false` | 是否启用仅 prefill 阶段的 sequence parallel；支持 `enable_chunked_prefill=true`，但仅限纯 prefill batch（`PREFILL` / `CHUNKED_PREFILL`）。 | +| `enable_mm_encoder_dp` | `bool` | `false` | 是否为多模态模型启用 encoder data parallel。 | +| `enable_multi_stream_parallel` | `bool` | `false` | 是否在 prefill 阶段通过双 stream 和双 micro batch 启用计算通信并行；详见 [多流并行](/zh/features/multi_streams/)。 | +| `micro_batch_num` | `int32` | `1` | 多流并行使用的 micro batch 数。 | +| `enable_dp_balance` | `bool` | `false` | 是否启用 DP 负载均衡；启用后会 shuffle 单个 DP batch 内的 sequences。 | + +## EPLBConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `enable_eplb` | `bool` | `false` | 是否启用 expert parallel load balance；详见 [EPLB](/zh/features/eplb/)。 | +| `redundant_experts_num` | `int32` | `1` | 每个 device 上的冗余 expert 数量。 | +| `eplb_update_interval` | `int64` | `1000` | EPLB 更新间隔。 | +| `eplb_update_threshold` | `double` | `0.8` | EPLB 更新阈值。 | +| `expert_parallel_degree` | `int32` | `0` | Expert parallel degree。 | +| `rank_tablefile` | `string` | `""` | ATB HCCL rank table 文件。 | + +## DistributedConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `master_node_addr` | `string` | `"127.0.0.1:19888"` | 多机分布式服务的 master 地址，例如 `10.18.1.1:9999`。 | +| `xtensor_master_node_addr` | `string` | `"127.0.0.1:19889"` | XTensor 分布式服务的 master 地址，例如 `10.18.1.1:9999`。 | +| `nnodes` | `int32` | `1` | 多机节点数量。 | +| `node_rank` | `int32` | `0` | 当前节点 rank。 | +| `etcd_addr` | `string` | `""` | 保存实例元信息的 etcd 地址。 | +| `etcd_namespace` | `string` | `""` | xLLM etcd key 使用的可选 namespace 前缀，例如 `prod-a`。 | +| `enable_service_routing` | `bool` | `false` | 是否启用 xLLM service routing。 | +| `heart_beat_interval` | `double` | `0.5` | 心跳间隔。 | +| `etcd_ttl` | `int32` | `3` | etcd key 的 TTL。 | + +## DisaggPDConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `enable_disagg_pd` | `bool` | `false` | 是否启用 Prefill-Decode 分离执行；详见 [PD 分离](/zh/features/disagg_pd/)。 | +| `enable_pd_ooc` | `bool` | `false` | 是否在 PD 分离模式下启用在线/离线混部。 | +| `disagg_pd_port` | `int32` | `7777` | PD 分离 bRPC server 的监听端口。 | +| `instance_role` | `string` | `"DEFAULT"` | 当前实例角色，例如 `DEFAULT`、`PREFILL`、`DECODE`、`MIX`。 | +| `kv_cache_transfer_type` | `string` | `"LlmDataDist"` | KV Cache 传输类型，例如 `LlmDataDist`、`Mooncake`、`HCCL`。 | +| `kv_cache_transfer_mode` | `string` | `"PUSH"` | KV Cache 传输模式，例如 `PUSH`、`PULL`。 | +| `transfer_listen_port` | `int32` | `26000` | KV Cache Transfer 的监听端口。 | +| `kv_push_dst_rotate` | `bool` | `false` | 在 `push_kv_blocks` 中按 KV-split rank 轮转遍历目标 worker，用于分散对 decode worker 的流量。 | + +## SpeculativeConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `draft_model` | `string` | `""` | draft 模型路径；MTP 使用方式详见 [MTP](/zh/features/mtp/)。 | +| `draft_devices` | `string` | `""` | draft 模型使用的设备，例如 `npu:0`、`npu:0,npu:1`；未指定时，启用 speculative decoding 会使用 target 模型的设备。 | +| `num_speculative_tokens` | `int32` | `0` | 每轮 speculative decoding 生成的 speculative token 数。 | +| `speculative_algorithm` | `string` | `"MTP"` | Speculative decoding 算法，支持 `MTP`、`Eagle3`、`Suffix`。 | +| `speculative_suffix_cache_max_depth` | `int32` | `64` | Suffix speculative decoding 的后缀树最大深度。 | +| `speculative_suffix_max_spec_factor` | `double` | `1.0` | Suffix speculation 相对于匹配长度的最大 token 系数。 | +| `speculative_suffix_max_spec_offset` | `double` | `0.0` | Suffix speculation 的最大 token 加性偏移。 | +| `speculative_suffix_min_token_prob` | `double` | `0.1` | Suffix speculation 使用的最小 token 概率。 | +| `speculative_suffix_max_cached_requests` | `int32` | `-1` | Suffix speculation 全局最大缓存请求数；`-1` 表示不限，`0` 表示禁用。 | +| `speculative_suffix_use_tree_spec` | `bool` | `false` | 是否使用 tree-based suffix speculation，而不是 path speculation。 | +| `enable_opt_validate_probs` | `bool` | `false` | validate 阶段是否直接使用 selected-only `draft_probs [B,S]`；设为 `false` 时会将 selected-only cache 值恢复为 dense `[B,S,V]`。 | +| `enable_atb_spec_kernel` | `bool` | `false` | 是否使用 ATB speculative kernel。 | + +## ProfileConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `enable_profile_step_time` | `bool` | `false` | 是否启用 step time profiling。 | +| `enable_profile_token_budget` | `bool` | `false` | 是否启用 token budget profiling。 | +| `enable_latency_aware_schedule` | `bool` | `false` | 是否使用预测 latency 进行 latency-aware schedule。 | +| `profile_max_prompt_length` | `int32` | `2048` | profiling 使用的最大 prompt 长度。 | +| `max_global_ttft_ms` | `int32` | `std::numeric_limits::max()` | 全局 TTFT 阈值，单位毫秒。 | +| `max_global_tpot_ms` | `int32` | `std::numeric_limits::max()` | 全局 TPOT 阈值，单位毫秒。 | +| `enable_profile_kv_blocks` | `bool` | `true` | profiling 时是否生成 KV Cache blocks。 | +| `disable_ttft_profiling` | `bool` | `false` | 是否禁用 TTFT profiling。 | +| `enable_forward_interruption` | `bool` | `false` | 是否启用 forward interruption。 | +| `enable_online_profile` | `bool` | `false` | 是否启用在线 timeline profiling 端点（`/start_profile` 和 `/stop_profile`）；目前仅支持 CUDA，需配合以 `nsys --capture-range=cudaProfilerApi` 启动 server。 | +| `profile_backend` | `string` | `"torch"` | 在线 profiling 后端。`torch` 在进程内记录 CPU+CUDA 活动，并在 `/stop_profile` 时写出 Chrome trace，无需外部 profiler；`cuda` 仅切换 CUDA profiler 的 capture range，需配合以 `nsys --capture-range=cudaProfilerApi` 启动。 | +| `profile_dir` | `string` | `""` | `torch` 在线 profiling 后端写出 timeline trace 的目录；为空表示当前工作目录。 | + +## ExecutionConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `enable_graph` | `bool` | `false` | 是否在 decode 阶段启用图执行以降低 kernel launch 开销和设备空闲时间；支持 CUDA Graph、ACL Graph（NPU）、MLU Graph 和 DCU Graph，详见 [图执行](/zh/features/graph_mode/)。 | +| `enable_graph_double_buffer` | `bool` | `true` | 是否为 NPU schedule-overlap decode 启用双缓冲 ACL graph 持久化参数与 graph 实例。 | +| `enable_graph_mode_decode_no_padding` | `bool` | `false` | decode 阶段是否按实际 `num_tokens` 捕获 graph，而不是按 padding 后 shape 捕获。 | +| `enable_prefill_piecewise_graph` | `bool` | `false` | 是否在 prefill 阶段启用 piecewise CUDA graph；attention 使用 eager 模式，其他操作捕获进 CUDA graph。 | +| `enable_graph_vmm_pool` | `bool` | `true` | 是否启用 VMM-backed CUDA graph memory pool，用于多 shape graph 的显存复用。 | +| `max_tokens_for_graph_mode` | `int32` | `2048` | 图执行最大 token 数；`0` 表示不限制。 | +| `acl_graph_decode_batch_size_limit` | `int32` | `16` | NPU 上 ACL graph 的 decode batch size 阈值；实际 decode batch size 超过该值时，ACL graph decode 会回退到 eager 模式以避免 OOM。 | +| `enable_shm` | `bool` | `false` | 是否为模型执行启用共享内存。 | +| `use_contiguous_input_buffer` | `bool` | `true` | 是否使用连续的 device input buffer 执行模型。 | +| `input_shm_size` | `uint64` | `1024` | 输入共享内存大小，默认 1GB。 | +| `output_shm_size` | `uint64` | `128` | 输出共享内存大小，默认 128MB。 | +| `random_seed` | `int32` | `-1` | 随机数生成器 seed；`-1` 表示不固定 seed。 | + +## KernelConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `enable_customize_mla_kernel` | `bool` | `false` | 是否启用自定义 MLA kernel。 | +| `npu_kernel_backend` | `string` | `"AUTO"` | NPU kernel 后端，支持 `AUTO`、`ATB`、`TORCH`。 | +| `enable_intralayer_addnorm` | `bool` | `false` | 是否启用 fused intralayer addnorm ops。 | +| `enable_fused_mc2` | `int32` | `-1` | NPU 的 Fused MC2 模式；`-1` 使用自动默认值，`0` 禁用 fused MC2，正值启用 dense matmul-allreduce，`1` 对 MoE 使用 DispatchFFNCombine，`2` 对 MoE 使用 DispatchGmmCombineDecode。 | +| `enable_interlayer_addnorm` | `bool` | `false` | 是否启用 fused interlayer addnorm ops。 | +| `enable_split_rmsnorm_rope` | `bool` | `false` | 是否启用 fused split rmsnorm rope ops。 | +| `enable_aclnn_matmul` | `bool` | `false` | 是否为支持的 NPU ATB layer 启用 ACLNN matmul 后端。 | +| `enable_aclnn_swiglu` | `bool` | `false` | 是否为支持的 NPU ATB layer 启用 ACLNN SwiGLU 后端。 | + +## DiTConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `max_requests_per_batch` | `int32` | `1` | 每个 batch 的最大 request 数。 | +| `dit_cache_policy` | `string` | `"TaylorSeer"` | DiT cache 策略，例如 `None`、`FBCache`、`TaylorSeer`、`FBCacheTaylorSeer`、`ResidualCache`。 | +| `dit_cache_warmup_steps` | `int64` | `0` | warmup step 数量。 | +| `dit_cache_n_derivatives` | `int64` | `3` | TaylorSeer 使用的 derivative 数量。 | +| `dit_cache_skip_interval_steps` | `int64` | `3` | derivative 计算的跳步间隔。 | +| `dit_cache_residual_diff_threshold` | `double` | `0.09` | cache reuse 的 residual difference 阈值。 | +| `dit_cache_start_steps` | `int64` | `5` | 起始阶段跳过的 step 数。 | +| `dit_cache_end_steps` | `int64` | `5` | 末尾阶段跳过的 step 数。 | +| `dit_cache_start_blocks` | `int64` | `5` | 起始阶段跳过的 block 数。 | +| `dit_cache_end_blocks` | `int64` | `5` | 末尾阶段跳过的 block 数。 | +| `dit_sp_communication_overlap` | `bool` | `true` | 是否为 sequence parallel 启用通信与计算的 overlap。 | +| `dit_debug_print` | `bool` | `false` | 是否打印 DiT 模型 debug 信息。 | +| `dit_generation_image_area_max` | `int64` | `0` | 图像生成请求允许的最大图像面积（宽 * 高）；`0` 表示不限制。 | +| `dit_enable_vae_tiling` | `bool` | `false` | 是否启用 VAE tiling，目前仅支持 `qwen-image-edit-plus`。 | +| `dit_vae_image_size` | `int64` | `1048576` | Qwen-Image-Edit-Plus VAE 用于计算维度的 image size。 | +| `dit_sparse_attention_enabled` | `bool` | `false` | 是否为 WAN 启用 block-wise sparse attention / RainFusion。 | +| `dit_sparse_attention_sparsity` | `double` | `0.5` | sparse attention 稀疏度，取值 `[0.0, 1.0)`；`0.0` 表示 dense attention，`0.5` 表示丢弃 50% 的 block。 | +| `dit_sparse_attention_pool_size` | `int64` | `128` | block-wise mask 生成时 sparse attention 的 pooling 窗口大小。 | +| `dit_sparse_attention_sparse_start_step` | `int64` | `0` | 开始使用 sparse attention 的 step 索引；此前的 step 使用 dense attention。 | +| `dit_sparse_attention_version` | `string` | `"rain_fusion"` | sparse attention 版本：`rain_fusion`（frame-pairing + `aclnnRainFusionAttention`）或 `sparse_attention`（block-decompose + `aclnnBlockSparseAttention`）。 | +| `dit_sparse_attention_mask_refresh_steps` | `int64` | `1` | 每 N 个 diffusion step 重新计算一次 block sparse mask；`1` 表示每步都算，值越大 mask 复用越久。 | + +## RecConfig + +| 参数名称 | 类型 | 默认值 | 参数含义 | +|:---------|:-----|:-------|:---------| +| `enable_rec_fast_sampler` | `bool` | `true` | 是否为 Rec pipeline 启用 RecSampler fast sampling path。 | +| `enable_rec_prefill_only` | `bool` | `false` | 是否启用 Rec prefill-only 模式，不分配 decoder self-attention blocks。 | +| `enable_xattention_one_stage` | `bool` | `false` | 是否在 Rec multi-round 模式下强制使用 xattention one-stage decode。 | +| `max_decode_rounds` | `int32` | `0` | multi-step decoding 的最大 decode round 数；`0` 表示禁用。 | +| `enable_constrained_decoding` | `bool` | `false` | 是否启用 constrained decoding，用预定义规则约束输出格式或结构。 | +| `output_rec_logprobs` | `bool` | `false` | 是否输出 Rec multi-round token-aligned logprobs；启用后缺失的 per-token logprobs 会用最终 beam logprob 填充。 | +| `enable_convert_tokens_to_item` | `bool` | `false` | 是否在 REC/OneRec response 中将 token ids 转换为 item id。 | +| `enable_output_sku_logprobs` | `bool` | `false` | 是否输出 REC/OneRec token-aligned logprobs tensor。 | +| `enable_extended_item_info` | `bool` | `false` | 是否解析并输出 REC extended item info tensors。 | +| `each_conversion_threshold` | `int32` | `50` | 每个 REC token triplet 最多输出的 item 数。 | +| `total_conversion_threshold` | `int32` | `1000` | 单个 REC response 最多输出的 item 总数。 | +| `request_queue_size` | `int32` | `100000` | scheduler request queue 大小。 | +| `rec_worker_max_concurrency` | `uint32` | `1` | Rec worker 并行执行并发度；小于等于 `1` 表示禁用并发 Rec worker。 | diff --git a/docs/src/content/docs/zh/cookbook.md b/docs/src/content/docs/zh/cookbook.md new file mode 100644 index 0000000000..5fddf379d2 --- /dev/null +++ b/docs/src/content/docs/zh/cookbook.md @@ -0,0 +1,13 @@ +--- +title: "实践指南" +description: "xLLM 实践示例和部署案例" +--- + +本章节用于汇总 xLLM 的实践示例、部署案例和常见推理场景排障说明。 + +后续会随着实践内容整理逐步补充。 + +## 章节 + +- [自回归模型](/zh/cookbook/autoregressive_models/) +- [扩散模型](/zh/cookbook/diffusion_models/) diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models.md b/docs/src/content/docs/zh/cookbook/autoregressive_models.md new file mode 100644 index 0000000000..c344450f8b --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models.md @@ -0,0 +1,16 @@ +--- +title: "自回归模型" +description: "xLLM 自回归模型推理实践" +--- + +本章节用于汇总使用 xLLM 服务自回归模型的实践指南。 + +后续会随着实践内容整理逐步补充。 + +## 模型系列 + +- [Qwen](/zh/cookbook/autoregressive_models/qwen/qwen3_5/) +- [DeepSeek](/zh/cookbook/autoregressive_models/deepseek/deepseek_v4/) +- [GLM](/zh/cookbook/autoregressive_models/glm/glm_5/) +- [Kimi](/zh/cookbook/autoregressive_models/kimi/kimi2/) +- [MinMax](/zh/cookbook/autoregressive_models/minmax/minmax_m2_7/) diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_r1.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_r1.md new file mode 100644 index 0000000000..92f3adeb0d --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_r1.md @@ -0,0 +1,8 @@ +--- +title: "DeepSeek-R1" +description: "DeepSeek-R1 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 DeepSeek-R1 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v3.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v3.md new file mode 100644 index 0000000000..5f834cbb11 --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v3.md @@ -0,0 +1,8 @@ +--- +title: "DeepSeek-V3" +description: "DeepSeek-V3 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 DeepSeek-V3 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v3_1.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v3_1.md new file mode 100644 index 0000000000..0527106c87 --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v3_1.md @@ -0,0 +1,8 @@ +--- +title: "DeepSeek-V3.1" +description: "DeepSeek-V3.1 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 DeepSeek-V3.1 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v3_2.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v3_2.md new file mode 100644 index 0000000000..6b07973cef --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v3_2.md @@ -0,0 +1,8 @@ +--- +title: "DeepSeek-V3.2" +description: "DeepSeek-V3.2 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 DeepSeek-V3.2 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md new file mode 100644 index 0000000000..0e2be15dec --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md @@ -0,0 +1,269 @@ +--- +title: "DeepSeek-V4" +description: "DeepSeek-V4 在 Ascend A3 设备上的 xLLM 推理实践指南" +--- +# 使用 xLLM 在 Ascend A3 设备 推理 + +源码地址：https://github.com/jd-opensource/xllm + +国内可用: https://gitcode.com/xLLM-AI/xllm + +权重下载 + +Flash权重： +https://modelers.cn/models/Eco-Tech/DeepSeek-V4-Flash-w8a8-mtp + +Pro权重: +https://modelers.cn/models/Eco-Tech/DeepSeek-V4-Pro-w4a8-mtp + + +## 1. 拉取镜像环境 + +首先下载xLLM提供的镜像： + +```bash +# A2 x86 +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-x86-cann9-20260605 +# A2 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-arm-cann9-20260605 +# A3 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-cann9-20260605 +``` + +然后创建对应的容器 + +```bash +sudo docker run -it --ipc=host -u 0 --privileged --name mydocker --network=host \ + -v /var/queue_schedule:/var/queue_schedule \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ + -v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \ + -v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \ + -v /var/log/npu/slog/:/var/log/npu/slog \ + -v ~/.ssh:/root/.ssh \ + -v /var/log/npu/profiling/:/var/log/npu/profiling \ + -v /var/log/npu/dump/:/var/log/npu/dump \ + -v /runtime/:/runtime/ -v /etc/hccn.conf:/etc/hccn.conf \ + -v /export/home:/export/home \ + -v /home/:/home/ \ + -w /export/home \ + quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-cann9-20260605 +``` + +## 2. 拉取源码并编译 + +下载官方仓库与模块依赖： + +```bash +git clone https://github.com/jd-opensource/xllm +cd xllm +git submodule update --init --recursive +``` + +下载安装依赖: + +```bash +pip install --upgrade pre-commit +``` + +执行编译，在`build/`下生成可执行文件`build/xllm/core/server/xllm`： + +```bash +python setup.py build --device npu +``` + +## 3. 启动模型 + +### 若机器为重启后初次拉起服务，需先执行以下脚本对device进行初始化 + +> 若不执行且 npu 未初始化可能导致 xllm 进程拉起失败 + +```bash +python -c "import torch_npu +for i in range(16):torch_npu.npu.set_device(i)" +``` + +### 导出MTP权重 + +```bash +python tools/export_mtp.py --input-dir ${W4A8/W8A8权重目录} --output-dir ${导出MTP权重目录} +``` + +### 环境变量 + +```bash +##### 1， 配置依赖路径相关环境变量 + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source ${ASCEND_TOOLKIT_HOME}/opp/vendors/custom_xllm_math/bin/set_env.bash + +##### 2， 配置日志相关环境变量 +rm -rf /root/ascend/log/ +rm -rf core.* + +##### 3. 配置性能、通信相关环境变量 +export HCCL_IF_BASE_PORT=43432 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export NPU_MEMORY_FRACTION=0.96 +export ATB_WORKSPACE_MEM_ALLOC_ALG_TYPE=3 +export ATB_WORKSPACE_MEM_ALLOC_GLOBAL=1 +export ATB_LAYER_INTERNAL_TENSOR_REUSE=1 +export ATB_CONTEXT_WORKSPACE_SIZE=0 +export OMP_NUM_THREADS=12 +export ALLOW_INTERNAL_FORMAT=1 + +``` + +## 启动命令 - 单机拉起样例 + +```bash +BATCH_SIZE=256 +#推理最大batch数量 +XLLM_PATH="./myxllm/xllm/build/xllm/core/server/xllm" +#推理入口文件路径（上一步中编译产物） +MODEL_PATH=/path/to/dsv4 +#模型路径 +DRAFT_MODEL_PATH=/path/to/dsv4_mtp +#导出的mtp权重 + +MASTER_NODE_ADDR="11.87.49.110:10015" +LOCAL_HOST="11.87.49.110" +# Service Port +START_PORT=18994 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=8 + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + nohup $XLLM_PATH -model-id ds \ + --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + --max_memory_utilization=0.9 \ + --max_tokens_per_batch=2048 \ + --max_seqs_per_batch=32 \ + --block_size=128 \ + --communication_backend="hccl" \ + --tool_call_parser=deepseekv4 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=true \ + --enable_schedule_overlap=true \ + --enable_graph=true \ + --npu_kernel_backend=TORCH \ + --ep_size=8 \ + --dp_size=2 \ + > $LOG_FILE 2>&1 & +done + + # 开启mtp时需要的变量 + # --draft_model=$DRAFT_MODEL_PATH \ + # --draft_devices="npu:$DEVICE" \ + # --num_speculative_tokens=1 \ + +# numactl -C xxxxx 亲和性绑核(NUMA亲和性查询命令： npu-smi info -t topo) +#--max_memory_utilization 单卡最大显存占用比例 +#--max_tokens_per_batch 单batch最大token数 （主要限制prefill） +#--max_seqs_per_batch 单batch最大请求数 （主要限制decoe） +#--communication_backend 通信backend 可选(hccl / lccl) 此处建议hccl +#--enable_schedule_overlap 开启异步调度 +#--enable_prefix_cache 开启prefix_cache +#--enable_chunked_prefill 开启chunked_prefill +#--enable_graph 开启aclgraph +#--draft_model mtp - mtp权重路径 +#--draft_devices mtp - mtp推理设备(与主模型同一) +#--num_speculative_tokens mtp - 预测token数 +``` + +日志出现"Brpc Server Started"表示服务成功拉起。 + +## 其他可选环境变量 + +```bash +#开启确定性计算 +export LCCL_DETERMINISTIC=1 +export HCCL_DETERMINISTIC=true +export ATB_MATMUL_SHUFFLE_K_ENABLE=0 + +# #开启动态profiling模式 +# export PROFILING_MODE=dynamic +# \rm -rf ~/dynamic_profiling_socket_* +``` + +## 启动命令 - 双机拉起样例 + +### Node0 (master) + +```bash +MASTER_NODE_ADDR="11.87.49.110:19990" +LOCAL_HOST="11.87.49.110" +START_PORT=15890 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=32 +LOCAL_NODES=16 +export HCCL_IF_BASE_PORT=48439 +unset HCCL_OP_EXPANSION_MODE + +for (( i=0; i<$LOCAL_NODES; i++ )); do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + nohup $XLLM_PATH \ + --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + ...... + --rank_tablefile=/yourPath/ranktable.json \ + > $LOG_FILE 2>&1 & +done +``` + +#### Node1 (worker) + +```bash +MASTER_NODE_ADDR="11.87.49.110:19990" +LOCAL_HOST="11.87.49.111" +START_PORT=15890 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=32 +LOCAL_NODES=16 +export HCCL_IF_BASE_PORT=48439 +unset HCCL_OP_EXPANSION_MODE + +for (( i=0; i<$LOCAL_NODES; i++ )); do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + nohup $XLLM_PATH \ + --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$((i + LOCAL_NODES)) \ + ...... + --rank_tablefile=/yourPath/ranktable.json \ + > $LOG_FILE 2>&1 & +done +``` + +### ranktable样例 + + [A3 ranktable配置](https://www.hiascend.com/document/detail/zh/canncommercial/900/API/hcclug/hcclug_000066.html) + + [A2 ranktable配置](https://www.hiascend.com/document/detail/zh/canncommercial/900/API/hcclug/hcclug_000067.html) + + （注意A3与A2的ranktable格式差异） diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_5.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_5.md new file mode 100644 index 0000000000..e93f98974a --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_5.md @@ -0,0 +1,8 @@ +--- +title: "GLM-4.5" +description: "GLM-4.5 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 GLM-4.5 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_5v.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_5v.md new file mode 100644 index 0000000000..1de01be1da --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_5v.md @@ -0,0 +1,8 @@ +--- +title: "GLM-4.5V" +description: "GLM-4.5V 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 GLM-4.5V 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_6.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_6.md new file mode 100644 index 0000000000..265714e618 --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_6.md @@ -0,0 +1,8 @@ +--- +title: "GLM-4.6" +description: "GLM-4.6 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 GLM-4.6 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_6v.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_6v.md new file mode 100644 index 0000000000..aaf0e7017b --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_6v.md @@ -0,0 +1,8 @@ +--- +title: "GLM-4.6V" +description: "GLM-4.6V 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 GLM-4.6V 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_7.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_7.md new file mode 100644 index 0000000000..bec58ac02f --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_7.md @@ -0,0 +1,8 @@ +--- +title: "GLM-4.7" +description: "GLM-4.7 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 GLM-4.7 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_7_flash.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_7_flash.md new file mode 100644 index 0000000000..e45d03234a --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_4_7_flash.md @@ -0,0 +1,8 @@ +--- +title: "GLM-4.7-Flash" +description: "GLM-4.7-Flash 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 GLM-4.7-Flash 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md new file mode 100644 index 0000000000..1037900c5c --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md @@ -0,0 +1,524 @@ +--- +title: "GLM-5 / GLM-5.1 / GLM-5.2" +sidebar: + order: 2 +--- + ++ 源码地址：https://github.com/xLLM-AI/xllm + ++ 国内可用: https://gitcode.com/xLLM-AI/xllm + ++ 权重下载: + + [modelscope-GLM-5-W8A8](https://www.modelscope.cn/models/Eco-Tech/GLM-5-W8A8-xLLM-0403/files) + + [modelscope-GLM-5.1-W8A8](https://www.modelscope.cn/models/Eco-Tech/GLM-5.1-W8A8-xLLM/files) + + [modelscope-GLM-5.1-W4A8](https://www.modelscope.cn/models/Eco-Tech/GLM-5.1-w4a8) + + [modelscope-GLM-5.2-W8A8](https://www.modelscope.cn/models/Eco-Tech/GLM-5.2-W8A8/files) + +## 1.拉取镜像环境 + +首先下载xLLM提供的镜像： + +```bash +# A2 x86 +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-x86-cann9-20260605 +# A2 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-arm-cann9-20260605 +# A3 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-cann9-20260605 +``` + +**注意**: A2 机器性能未进行压测。 + +然后创建对应的容器 + +```bash +sudo docker run -it --ipc=host -u 0 --privileged --name mydocker --network=host \ + -v /var/queue_schedule:/var/queue_schedule \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ + -v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \ + -v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \ + -v /var/log/npu/slog/:/var/log/npu/slog \ + -v ~/.ssh:/root/.ssh \ + -v /var/log/npu/profiling/:/var/log/npu/profiling \ + -v /var/log/npu/dump/:/var/log/npu/dump \ + -v /runtime/:/runtime/ -v /etc/hccn.conf:/etc/hccn.conf \ + -v /export/home:/export/home \ + -v /home/:/home/ \ + -w /export/home \ + quay.io/jd_xllm/xllm-ai:xllm-dev-hb-rc2-x86 +``` + +## 2.拉取源码并编译 + +下载官方仓库与模块依赖： + +```bash +git clone https://github.com/xLLM-AI/xllm.git +cd xllm +git checkout release/v0.10.0 +git submodule update --init --recursive +``` + +下载安装依赖: + +```bash +pip install --upgrade pre-commit +yum install numactl +``` + +执行编译，在`build/`下生成可执行文件`build/xllm/core/server/xllm`： + +```bash +python setup.py build --device npu +``` + +## 3.启动模型 + +### 若机器为重启后初次拉起服务，需先执行以下脚本对device进行初始化 + +#若不执行且npu未初始化可能导致xllm进程拉起失败 + +```bash +python -c "import torch_npu +for i in range(16):torch_npu.npu.set_device(i)" +``` + +### 导出MTP权重 + +```bash +python tools/export_mtp.py --input-dir ${W4A8/W8A8权重目录} --output-dir ${导出MTP权重目录} +``` + +### 启动准备 + +```bash +##### 1， 配置相关环境变量 +export LD_PRELOAD=/usr/lib64/libtcmalloc.so.4:$LD_PRELOAD +export HCCL_EXEC_TIMEOUT=300 +export HCCL_CONNECT_TIMEOUT=300 +export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_IF_BASE_PORT=2864 + +##### 2， 清除残留日志 +rm -rf /root/ascend/log/ +``` + +## 启动命令 - A3单机 - GLM-5.2-W8A8 + +```bash +XLLM_PATH="./myxllm/xllm/build/xllm/core/server/xllm" +# xllm可执行文件路径 +MODEL_PATH=/path/to/GLM-5.2-W8A8/ +# 模型路径（以Glm-5.2-w8a8为例） +DRAFT_MODEL_PATH=/path/to/GLM-5.2-MTP/ +# 前面导出的mtp权重 + +MASTER_NODE_ADDR="11.87.49.110:10015" +LOCAL_HOST="11.87.49.110" +# Service Port +START_PORT=18994 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=16 + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + #可选：numactl绑核 (NUMA亲和性查询命令： npu-smi info -t topo) + #nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + nohup $XLLM_PATH \ + --model $MODEL_PATH \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + --max_memory_utilization=0.86 \ + --max_tokens_per_batch=4096 \ + --max_seqs_per_batch=16 \ + --block_size=128 \ + --enable_prefix_cache=true \ + --enable_chunked_prefill=true \ + --enable_graph=true \ + --enable_schedule_overlap=true \ + --communication_backend="hccl" \ + --graph_decode_batch_size_limit=2 \ + --draft_model=$DRAFT_MODEL_PATH \ + --draft_devices="npu:$DEVICE" \ + --num_speculative_tokens=3 \ + --ep_size=16 \ + --dp_size=2 \ + --tool_call_parser=auto \ + > $LOG_FILE 2>&1 & +done + +# --max_memory_utilization 单卡最大显存占用比例 +# --max_tokens_per_batch 单batch最大token数 （主要限制prefill） +# --max_seqs_per_batch 单batch最大请求数 （主要限制decode） +# --communication_backend 通信backend 可选(hccl / lccl) 此处建议hccl +# --enable_schedule_overlap 开启异步调度 +# --enable_prefix_cache 开启prefix_cache +# --enable_chunked_prefill 开启chunked_prefill +# --enable_graph 开启aclgraph，需要额外显存 +# --acl_graph_decode_batch_size_limit 抓图的最大bs，当前需<= 32 / (预测token数 + 1) +# --draft_model mtp - mtp权重路径 +# --draft_devices mtp - mtp推理设备(与主模型同一) +# --num_speculative_tokens mtp - 预测token数 +``` + +日志出现"Brpc Server Started"表示服务成功拉起。 + +## 其他可选环境变量 + +```bash +#开启确定性计算 +export LCCL_DETERMINISTIC=1 +export HCCL_DETERMINISTIC=true +export ATB_MATMUL_SHUFFLE_K_ENABLE=0 + +# 开启动态profiling模式 +export PROFILING_MODE=dynamic +\rm -rf ~/dynamic_profiling_socket_* +``` + +## 启动命令 - A3双机拉起样例 + +### Node0 (master) + +```bash +MASTER_NODE_ADDR="11.87.49.110:19990" +LOCAL_HOST="11.87.49.110" +START_PORT=15890 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=32 +LOCAL_NODES=16 +export HCCL_IF_BASE_PORT=48439 +unset HCCL_OP_EXPANSION_MODE + +for (( i=0; i<$LOCAL_NODES; i++ ))do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + --max_memory_utilization=0.85 \ + --max_tokens_per_batch=8192 \ + --max_seqs_per_batch=128 \ + --block_size=128 \ + --enable_prefix_cache=true \ + --enable_chunked_prefill=true \ + --communication_backend="hccl" \ + --enable_schedule_overlap=true \ + --enable_graph=true \ + --acl_graph_decode_batch_size_limit=4 \ + --draft_model=$DRAFT_MODEL_PATH \ + --draft_devices="npu:$DEVICE" \ + --num_speculative_tokens=3 \ + --ep_size=32 \ + --dp_size=4 \ + --rank_tablefile=/yourPath/ranktable.json \ + --tool_call_parser=auto \ + > $LOG_FILE 2>&1 & +done +``` + +### Node1 (worker) + +```bash +MASTER_NODE_ADDR="11.87.49.110:19990" +LOCAL_HOST="11.87.49.111" +START_PORT=15890 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=32 +LOCAL_NODES=16 +export HCCL_IF_BASE_PORT=48439 +unset HCCL_OP_EXPANSION_MODE + +for (( i=0; i<$LOCAL_NODES; i++ ))do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$((i + LOCAL_NODES)) \ + --max_memory_utilization=0.85 \ + --max_tokens_per_batch=8192 \ + --max_seqs_per_batch=128 \ + --block_size=128 \ + --enable_prefix_cache=true \ + --enable_chunked_prefill=true \ + --communication_backend="hccl" \ + --enable_schedule_overlap=true \ + --enable_graph=true \ + --acl_graph_decode_batch_size_limit=4 \ + --draft_model=$DRAFT_MODEL_PATH \ + --draft_devices="npu:$DEVICE" \ + --num_speculative_tokens=3 \ + --ep_size=32 \ + --dp_size=4 \ + --rank_tablefile=/yourPath/ranktable.json \ + --tool_call_parser=auto \ + > $LOG_FILE 2>&1 & +done +``` + +### ranktable样例 + + [A3 ranktable配置](https://www.hiascend.com/document/detail/zh/canncommercial/900/API/hcclug/hcclug_000066.html) + + [A2 ranktable配置](https://www.hiascend.com/document/detail/zh/canncommercial/900/API/hcclug/hcclug_000067.html) + + （注意A3与A2的ranktable格式差异） + +## device NUMA亲和性查看 + +命令： + +```bash +npu-smi info -t topo +``` + +前述命令中 + +```bash +numactl -C $((DEVICE*12))-$((DEVICE*12+11)) +``` + +表示该进程绑在对应亲和的核上，可根据机器具体情况修改绑定的核id + +## EX3.Glm-5 权重量化 (GLM5.2 量化指导待更新) + +### 安装msmodelslim + +```bash +pip install transformers==5.2.0 + +git clone https://gitcode.com/Ascend/msmodelslim.git +cd msmodelslim +bash install.sh +``` +### 量化执行 +```bash +msmodelslim quant \ + --model_path ${MODEL_PATH} \ + --save_path ${SAVE_PATH} \ + --device npu:0 \ + --model_type GLM-5 \ + --quant_type w8a8 \ + --trust_remote_code True +``` + +## PD分离 + +### etcd\xllm-service 安装 + +#### PD分离部署 + +`xllm`支持PD分离部署，这需要与另一个开源库[xllm service](https://github.com/xLLM-AI/xllm-service)配套使用。 + +##### xLLM Service依赖 + +首先，我们下载安装`xllm service`，与安装编译`xllm`类似： + +```bash +git clone https://github.com/xLLM-AI/xllm-service.git +cd xllm-service +git submodule init +git submodule update +``` + +##### etcd安装 + +`xllm_service`依赖[etcd](https://github.com/etcd-io/etcd)，使用etcd官方提供的[安装脚本](https://github.com/etcd-io/etcd/releases)进行安装，其脚本提供的默认安装路径是`/tmp/etcd-download-test/etcd`，我们可以手动修改其脚本中的安装路径，也可以运行完脚本之后手动迁移： + +```bash +mv /tmp/etcd-download-test/etcd /path/to/your/etcd +``` + +##### xLLM Service编译 + +先应用patch: + +```bash +sh prepare.sh +``` + +再执行编译: + +```bash +mkdir -p build +cd build +cmake .. +make -j 8 +cd .. +``` + +!!! warning "可能的错误" + 这里能会遇到关于`boost-locale`和`boost-interprocess`的安装错误：`vcpkg-src/packages/boost-locale_x64-linux/include: No such file or directory`,`/vcpkg-src/packages/boost-interprocess_x64-linux/include: No such file or directory` + 我们使用`vcpkg`重新安装这些包: + ```bash + /path/to/vcpkg remove boost-locale boost-interprocess + /path/to/vcpkg install boost-locale:x64-linux + /path/to/vcpkg install boost-interprocess:x64-linux + ``` + +### PD分离运行 + +启动etcd: + +```bash +./etcd-download-test/etcd --listen-peer-urls 'http://localhost:2390' --listen-client-urls 'http://localhost:2389' --advertise-client-urls 'http://localhost:2391' +``` + +跨机配置时，etcd参考如下： + +```bash +/tmp/etcd-download-test/etcd --listen-peer-urls 'http://0.0.0.0:3390' --listen-client-urls 'http://0.0.0.0:3389' --advertise-client-urls 'http://11.87.191.82:3389' +``` + +启动xllm service: + +```bash +ENABLE_DECODE_RESPONSE_TO_SERVICE=true ./xllm_master_serving --etcd_addr="127.0.0.1:12389" --http_server_port 28888 --rpc_server_port 28889 --tokenizer_path=/export/home/models/GLM-5-W8A8/ +``` + +跨机配置时，启动xllm service: + +```bash +ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_master_serving --etcd_addr="11.87.191.82:3389" --http_server_port 38888 --rpc_server_port 38889 --tokenizer_path=/export/home/models/GLM-5-W8A8/ +``` +- 启动Prefill实例 +```bash + BATCH_SIZE=256 + #推理最大batch数量 + XLLM_PATH="./myxllm/xllm/build/xllm/core/server/xllm" + #推理入口文件路径（上一步中编译产物） + MODEL_PATH=/export/home/models/GLM-5-w8a8/ + #模型路径（此处为int量化的Glm-5） + DRAFT_MODEL_PATH=/export/home/models/GLM-5-MTP/ + + MASTER_NODE_ADDR="11.87.49.110:10015" + LOCAL_HOST="11.87.49.110" + # Service Port + START_PORT=18994 + START_DEVICE=0 + LOG_DIR="logs" + NNODES=16 + + for (( i=0; i<$NNODES; i++ )) + do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + --model $MODEL_PATH --model_id glmmoe \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + --max_memory_utilization=0.86 \ + --max_tokens_per_batch=5000 \ + --max_seqs_per_batch=$BATCH_SIZE \ + --communication_backend=hccl \ + --enable_schedule_overlap=true \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_graph=true \ + --draft_model $DRAFT_MODEL_PATH \ + --draft_devices="npu:$DEVICE" \ + --num_speculative_tokens 1 \ + --tool_call_parser=auto \ + --enable_disagg_pd=true \ + --instance_role=PREFILL \ + --etcd_addr=$LOCAL_HOST:3389 \ + --transfer_listen_port=$((36100 + i)) \ + --disagg_pd_port=8877 \ + > $LOG_FILE 2>&1 & + done + + #--etcd_addr=$LOCAL_HOST:3389 参考etcd中advertise-client-urls的配置 + #--instance_role=DECODE PD配置，DECODE\PREFILL + ``` + +- 启动Decode实例 + + ```bash + BATCH_SIZE=256 + #推理最大batch数量 + XLLM_PATH="./myxllm/xllm/build/xllm/core/server/xllm" + #推理入口文件路径（上一步中编译产物） + MODEL_PATH=/export/home/models/GLM-5-w8a8/ + #模型路径（此处为int量化的Glm-5） + DRAFT_MODEL_PATH=/export/home/models/GLM-5-MTP/ + + MASTER_NODE_ADDR="11.87.49.110:10015" + LOCAL_HOST="11.87.49.110" + # Service Port + START_PORT=18994 + START_DEVICE=0 + LOG_DIR="logs" + NNODES=16 + + for (( i=0; i<$NNODES; i++ )) + do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + --model $MODEL_PATH --model_id glmmoe \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + --max_memory_utilization=0.86 \ + --max_tokens_per_batch=5000 \ + --max_seqs_per_batch=$BATCH_SIZE \ + --communication_backend=hccl \ + --enable_schedule_overlap=true \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_graph=true \ + --draft_model $DRAFT_MODEL_PATH \ + --draft_devices="npu:$DEVICE" \ + --num_speculative_tokens 1 \ + --tool_call_parser=auto \ + --enable_disagg_pd=true \ + --instance_role=DECODE \ + --etcd_addr=$LOCAL_HOST:3389 \ + --transfer_listen_port=$((36100 + i)) \ + --disagg_pd_port=8877 \ + > $LOG_FILE 2>&1 & + done + + #--etcd_addr=$LOCAL_HOST:3389 参考etcd中advertise-client-urls的配置 + #--instance_role=DECODE PD配置，DECODE\PREFILL + ``` + + 需要注意： + +- PD分离需要读取`/etc/hccn.conf`文件，确保将物理机上的该文件映射到了容器中 + +- `etcd_addr`需与`xllm_service`的`etcd_addr`相同 + 测试命令和上面类似，注意`curl http://localhost:{PORT}/v1/chat/completions ...`的`PORT`选择为启动xLLM service的`http_server_port`。 + +- 多机部署P或者Q时(例如部署两个P)，需要增加--rank_tablefile来完成通信。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5_1.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5_1.md new file mode 100644 index 0000000000..ad5011e7de --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5_1.md @@ -0,0 +1,6 @@ +--- +title: "GLM-5.1" +description: "GLM-5.1 推理实践已合并到 GLM-5 系列文档" +--- + +GLM-5.1 的推理实践已合并到 [GLM-5 / GLM-5.1 / GLM-5.2](/zh/cookbook/autoregressive_models/glm/glm_5/)。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5_2.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5_2.md new file mode 100644 index 0000000000..eceab63026 --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5_2.md @@ -0,0 +1,6 @@ +--- +title: "GLM-5.2" +description: "GLM-5.2 推理实践已合并到 GLM-5 系列文档" +--- + +GLM-5.2 的推理实践已合并到 [GLM-5 / GLM-5.1 / GLM-5.2](/zh/cookbook/autoregressive_models/glm/glm_5/)。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2.md new file mode 100644 index 0000000000..633071ed0b --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2.md @@ -0,0 +1,8 @@ +--- +title: "Kimi2" +description: "Kimi2 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 Kimi2 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md new file mode 100644 index 0000000000..67c50d772b --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md @@ -0,0 +1,300 @@ +--- +title: "Kimi-K2.5 / Kimi-K2.6" +sidebar: + order: 2 +--- + +- 源码地址：[https://github.com/xLLM-AI/xllm](https://github.com/xLLM-AI/xllm) +- 国内可用: [https://gitcode.com/xLLM-AI/xllm](https://gitcode.com/xLLM-AI/xllm) +- Kimi-K2.5 W8A8权重下载: [modelscope-Kimi-K2.5-W8A8-xLLM](https://www.modelscope.cn/models/Eco-Tech/Kimi-K2.5-W8A8-xLLM) +- Kimi-K2.6 W8A8权重下载: [modelscope-Kimi-K2.6-w8a8-xllm](https://www.modelscope.cn/models/Eco-Tech/Kimi-K2.6-w8a8-xllm) + +P.S. Kimi-K2.5 与 Kimi-K2.6 模型结构相同，后文以Kimi-K2.5 做例子说明整体部署流程 + +## 0.权重准备 + +### 从魔乐上下载权重 + +```bash +export MODELSCOPE_CACHE=path-to-model # 默认 ~/.cache/modelscope/hub +pip install modelscope +modelscope download --model Eco-Tech/Kimi-K2.5-W8A8-xLLM +``` + +## 1.拉取镜像环境 + +首先下载xLLM提供的镜像： + +```bash +# A3 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-20260429 +``` + +然后创建对应的容器 + +```bash +sudo docker run -it --ipc=host -u 0 --privileged --name xllm_kimi_k25 --network=host \ + -v /var/queue_schedule:/var/queue_schedule \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ + -v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \ + -v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \ + -v /var/log/npu/slog/:/var/log/npu/slog \ + -v ~/.ssh:/root/.ssh \ + -v /var/log/npu/profiling/:/var/log/npu/profiling \ + -v /var/log/npu/dump/:/var/log/npu/dump \ + -v /runtime/:/runtime/ -v /etc/hccn.conf:/etc/hccn.conf \ + -v /export/home:/export/home \ + -v /home/:/home/ \ + -w /export/home \ + quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-20260429 +``` + +## 2.拉取源码并编译 + +下载官方仓库与模块依赖： + +```bash +git clone https://github.com/xLLM-AI/xllm.git +cd xllm +git checkout main +git submodule init +git submodule update +``` + +下载安装依赖: + +```bash +pip install --upgrade pre-commit +yum install numactl +``` + +执行编译，在`build/`下生成可执行文件： + +```bash +python setup.py build +``` + +编译产物路径：`build/xllm/core/server/xllm` + +## 3.启动模型 + +### 若机器为重启后初次拉起服务，需先执行以下脚本对device进行初始化 + +若不执行且npu未初始化可能导致xllm进程拉起失败 + +```bash +python -c "import torch_npu +for i in range(16):torch_npu.npu.set_device(i)" +``` + +### 环境变量 + +```bash +##### 1. 配置依赖路径相关环境变量 +export PYTHON_INCLUDE_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])')" +export PYTHON_LIB_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])')" +export PYTORCH_NPU_INSTALL_PATH=/usr/local/libtorch_npu/ +export PYTORCH_INSTALL_PATH="$(python3 -c 'import torch, os; print(os.path.dirname(os.path.abspath(torch.__file__)))')" +export LIBTORCH_ROOT="$(python3 -c 'import torch, os; print(os.path.dirname(os.path.abspath(torch.__file__)))')" + +export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/xllm/op_api/lib/:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/usr/local/libtorch_npu/lib:$LD_LIBRARY_PATH +export LD_PRELOAD=/usr/lib64/libjemalloc.so.2:$LD_PRELOAD + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +##### 2. 配置日志相关环境变量 +rm -rf /root/atb/log/ +rm -rf /root/ascend/log/ +rm -rf core.* +export ASDOPS_LOG_LEVEL=ERROR +export ASDOPS_LOG_TO_STDOUT=1 +export ASDOPS_LOG_TO_FILE=1 + +##### 3. 配置性能、通信相关环境变量 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export NPU_MEMORY_FRACTION=0.96 +export ATB_WORKSPACE_MEM_ALLOC_ALG_TYPE=3 +export ATB_WORKSPACE_MEM_ALLOC_GLOBAL=1 + +export OMP_NUM_THREADS=12 +export ALLOW_INTERNAL_FORMAT=1 + +export ATB_LAYER_INTERNAL_TENSOR_REUSE=1 +export ATB_LLM_ENABLE_AUTO_TRANSPOSE=0 +export ATB_CONVERT_NCHW_TO_ND=1 +export ATB_LAUNCH_KERNEL_WITH_TILING=1 +export ATB_OPERATION_EXECUTE_ASYNC=2 +export ATB_CONTEXT_WORKSPACE_SIZE=0 +export INF_NAN_MODE_ENABLE=1 +export HCCL_EXEC_TIMEOUT=0 +export HCCL_CONNECT_TIMEOUT=7200 +export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_IF_BASE_PORT=2864 +``` + +## 启动命令 - Kimi_k25（双机 16卡32die tp=4, dp=8, ep=32） + +### Node0 (master) + +```bash +MASTER_NODE_ADDR="11.87.49.110:19990" +LOCAL_HOST="11.87.49.110" +START_PORT=15890 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=32 +LOCAL_NODES=16 +export HCCL_IF_BASE_PORT=48439 +unset HCCL_OP_EXPANSION_MODE + +for (( i=0; i<$LOCAL_NODES; i++ ))do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + --max_memory_utilization=0.85 \ + --max_tokens_per_batch=8192 \ + --max_seqs_per_batch=20 \ + --block_size=128 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --communication_backend="hccl" \ + --enable_schedule_overlap=true \ + --enable_graph=false \ + --enable_shm=true \ + --ep_size=32 \ + --dp_size=8 \ + --input_shm_size=4096 \ + --rank_tablefile=/yourPath/ranktable.json \ + > $LOG_FILE 2>&1 & +done +``` + +#### Node1 (worker) + +```bash +MASTER_NODE_ADDR="11.87.49.110:19990" +LOCAL_HOST="11.87.49.111" +START_PORT=15890 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=32 +LOCAL_NODES=16 +export HCCL_IF_BASE_PORT=48439 +unset HCCL_OP_EXPANSION_MODE + +for (( i=0; i<$LOCAL_NODES; i++ ))do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$((i + LOCAL_NODES)) \ + --max_memory_utilization=0.85 \ + --max_tokens_per_batch=8192 \ + --max_seqs_per_batch=20 \ + --block_size=128 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --communication_backend="hccl" \ + --enable_schedule_overlap=true \ + --enable_graph=false \ + --enable_shm=true \ + --ep_size=32 \ + --dp_size=8 \ + --input_shm_size=4096 \ + --rank_tablefile=/yourPath/ranktable.json \ +done +``` + +#### ranktable样例 + +ranktable配置指导：[https://www.hiascend.com/document/detail/zh/canncommercial/83RC1/hccl/hcclug/hcclug_000014.html](https://www.hiascend.com/document/detail/zh/canncommercial/83RC1/hccl/hcclug/hcclug_000014.html) + +```bash +ln -s /usr/local/Ascend/driver/tools/hccn_tool /usr/sbin/ + +#device_ip +for i in {0..15};do hccn_tool -i $i -vnic -g; done + +#super_device_id +for i in {0..7};do for j in {0..1}; do npu-smi info -t spod-info -i $i -c $j; done; done +``` + +```json +{ + "status": "completed", + "version": "1.2", + "server_count": "2", + "server_list": [ + { + "server_id": "10.87.191.98", + "host_nic_ip": "reserve", + "host_ip": "10.87.191.98", + "container_ip": "10.87.191.98", + "device": [ + { + "device_id": "0", + "device_ip": "192.24.2.199", + "super_device_id": "100663296", + "rank_id": "16" + }, + ... + { + "device_id": "15", + "device_ip": "192.24.3.184", + "super_device_id": "102563855", + "rank_id": "31" + } + ] + }, + { + "server_id": "10.87.191.102", + "host_nic_ip": "reserve", + "host_ip": "10.87.191.102", + "container_ip": "10.87.191.102", + "device": [ + { + "device_id": "0", + "device_ip": "192.28.2.199", + "super_device_id": "117440512", + "rank_id": "0" + }, + ... + { + "device_id": "15", + "device_ip": "192.28.3.184", + "super_device_id": "119341071", + "rank_id": "15" + } + ] + } + ], + "super_pod_list": [ + { + "super_pod_id": "2", + "server_list": [ + { + "server_id": "10.87.191.98" + }, + { + "server_id": "10.87.191.102" + } + ] + } + ] +} +``` + +日志出现"Application startup complete."表示服务成功拉起。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md new file mode 100644 index 0000000000..2fc02f1c05 --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md @@ -0,0 +1,198 @@ +--- +title: "MiniMax-M2.7" +sidebar: + order: 3 +--- + ++ 源码地址：https://github.com/xLLM-AI/xllm + ++ 国内可用: https://gitcode.com/xLLM-AI/xllm + ++ 权重下载: [modelscope-MiniMax-M2.7](https://www.modelscope.cn/models/MiniMax/MiniMax-M2.7) ++ 离线反量化权重: [modelscope-Minimax2.7-BF16-xLLM](https://modelscope.cn/models/Eco-Tech/Minimax2.7-BF16-xLLM) + +## 0.权重准备 + +MiniMax-M2.7 原始权重为 FP8 格式，xLLM 支持以下三种方式加载： + +### 方式一：直接加载 FP8 权重（在线反量化） + +直接使用原始 FP8 权重路径，xLLM 会在推理时在线将 FP8 反量化为 BF16 计算，无需额外处理。 + +```bash +MODEL_PATH=/path/to/MiniMax-M2.7/ +``` + +### 方式二：离线反量化 + +使用工具脚本预先将 FP8 权重转换为 BF16 格式，避免在线反量化的额外开销： + +```bash +python tools/dequant_minimax_fp8.py --input-dir /path/to/MiniMax-M2.7/ --output-dir /path/to/MiniMax-M2.7-bf16/ +``` + +### 方式三：下载预转换的 BF16 权重 + +直接下载已反量化好的 BF16 权重： + +```bash +git clone https://www.modelscope.cn/Eco-Tech/Minimax2.7-BF16-xLLM.git +``` + +## 1.拉取镜像环境 + +首先下载xLLM提供的镜像： + +```bash +# A3 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-20260429 +``` + +然后创建对应的容器 + +```bash +sudo docker run -it --ipc=host -u 0 --privileged --name xllm_minimax --network=host \ + -v /var/queue_schedule:/var/queue_schedule \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ + -v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \ + -v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \ + -v /var/log/npu/slog/:/var/log/npu/slog \ + -v ~/.ssh:/root/.ssh \ + -v /var/log/npu/profiling/:/var/log/npu/profiling \ + -v /var/log/npu/dump/:/var/log/npu/dump \ + -v /runtime/:/runtime/ -v /etc/hccn.conf:/etc/hccn.conf \ + -v /export/home:/export/home \ + -v /home/:/home/ \ + -w /export/home \ + quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-20260429 +``` + +## 2.拉取源码并编译 + +下载官方仓库与模块依赖： + +```bash +git clone https://github.com/xLLM-AI/xllm.git +cd xllm +git checkout preview/minimax-minimal +git submodule init +git submodule update +``` + +下载安装依赖: + +```bash +pip install --upgrade pre-commit +yum install numactl +``` + +执行编译，在`build/`下生成可执行文件： + +```bash +python setup.py build +``` + +编译产物路径：`build/xllm/core/server/xllm` + +## 3.启动模型 + +### 若机器为重启后初次拉起服务，需先执行以下脚本对device进行初始化 + +若不执行且npu未初始化可能导致xllm进程拉起失败 + +```bash +python -c "import torch_npu +for i in range(16):torch_npu.npu.set_device(i)" +``` + +### 环境变量 + +```bash +##### 1. 配置依赖路径相关环境变量 +export PYTHON_INCLUDE_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])')" +export PYTHON_LIB_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])')" +export PYTORCH_NPU_INSTALL_PATH=/usr/local/libtorch_npu/ +export PYTORCH_INSTALL_PATH="$(python3 -c 'import torch, os; print(os.path.dirname(os.path.abspath(torch.__file__)))')" +export LIBTORCH_ROOT="$(python3 -c 'import torch, os; print(os.path.dirname(os.path.abspath(torch.__file__)))')" + +export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/xllm/op_api/lib/:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/usr/local/libtorch_npu/lib:$LD_LIBRARY_PATH +export LD_PRELOAD=/usr/lib64/libjemalloc.so.2:$LD_PRELOAD + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +##### 2. 配置日志相关环境变量 +rm -rf /root/atb/log/ +rm -rf /root/ascend/log/ +rm -rf core.* +export ASDOPS_LOG_LEVEL=ERROR +export ASDOPS_LOG_TO_STDOUT=1 +export ASDOPS_LOG_TO_FILE=1 + +##### 3. 配置性能、通信相关环境变量 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export NPU_MEMORY_FRACTION=0.96 +export ATB_WORKSPACE_MEM_ALLOC_ALG_TYPE=3 +export ATB_WORKSPACE_MEM_ALLOC_GLOBAL=1 + +export OMP_NUM_THREADS=12 +export ALLOW_INTERNAL_FORMAT=1 + +export ATB_LAYER_INTERNAL_TENSOR_REUSE=1 +export ATB_LLM_ENABLE_AUTO_TRANSPOSE=0 +export ATB_CONVERT_NCHW_TO_ND=1 +export ATB_LAUNCH_KERNEL_WITH_TILING=1 +export ATB_OPERATION_EXECUTE_ASYNC=2 +export ATB_CONTEXT_WORKSPACE_SIZE=0 +export INF_NAN_MODE_ENABLE=1 +export HCCL_EXEC_TIMEOUT=0 +export HCCL_CONNECT_TIMEOUT=7200 +export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_IF_BASE_PORT=2864 +``` + +## 启动命令 - MiniMax-M2.7（单机 16卡 TP=16） + +```bash +BATCH_SIZE=256 +#推理最大batch数量 +XLLM_PATH="build/xllm/core/server/xllm" +#推理入口文件路径（上一步中编译产物） +MODEL_PATH=/path/to/MiniMax-M2.7/ +#模型路径 + +MASTER_NODE_ADDR="10.143.3.204:10015" +LOCAL_HOST="10.143.3.204" +# Service Port +START_PORT=18994 +START_DEVICE=0 +LOG_DIR="logs" +NNODES=16 + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --node_rank=$i \ + --max_memory_utilization=0.90 \ + --max_tokens_per_batch=8192 \ + --max_seqs_per_batch=$BATCH_SIZE \ + --communication_backend=hccl \ + --enable_chunked_prefill=false \ + --enable_prefix_cache=false \ + --enable_schedule_overlap=false \ + --enable_graph=false \ + --enable_atb_spec_kernel=false \ + > $LOG_FILE 2>&1 & +done +``` diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen2_5_vl.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen2_5_vl.md new file mode 100644 index 0000000000..d99613b869 --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen2_5_vl.md @@ -0,0 +1,8 @@ +--- +title: "Qwen2.5-VL" +description: "Qwen2.5-VL 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 Qwen2.5-VL 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3.md new file mode 100644 index 0000000000..b26bd1e9eb --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3.md @@ -0,0 +1,8 @@ +--- +title: "Qwen3" +description: "Qwen3 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 Qwen3 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md new file mode 100644 index 0000000000..39342f4493 --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md @@ -0,0 +1,158 @@ +--- +title: "Qwen3.5" +sidebar: + order: 1 +--- + ++ 源码地址：https://github.com/xLLM-AI/xllm + ++ 国内可用: https://gitcode.com/xLLM-AI/xllm + ++ 权重下载: [modelscope-Qwen3.5-27B](https://www.modelscope.cn/models/Qwen/Qwen3.5-27B) + +## 1.拉取镜像环境 + +首先下载xLLM提供的镜像： + +```bash +# A3 arm (CANN 9) +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-cann9-20260605 +``` + +然后创建对应的容器 + +```bash +docker run -it -d \ + --ipc=host \ + -u 0 \ + --privileged \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --name xllm_qwen35 \ + --network=host \ + --device=/dev/davinci0 \ + --device=/dev/davinci_manager \ + --device=/dev/devmm_svm \ + --device=/dev/hisi_hdc \ + -v /var/queue_schedule:/var/queue_schedule \ + -v /mnt/cfs/9n-das-admin/llm_models:/mnt/cfs/9n-das-admin/llm_models \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ + -v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \ + -v /usr/local/sbin/:/usr/local/sbin/ \ + -v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \ + -v /var/log/npu/slog/:/var/log/npu/slog \ + -v /var/log/npu/profiling/:/var/log/npu/profiling \ + -v /var/log/npu/dump/:/var/log/npu/dump \ + -v /export/home:/export/home \ + -v ~/.ssh:/root/.ssh \ + -v /home/:/home/ \ + -v /runtime/:/runtime/ \ + -w /home \ + quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-cann9-20260605 +``` + +## 2.拉取源码并编译 + +下载官方仓库与模块依赖： + +```bash +git clone https://github.com/xLLM-AI/xllm.git +cd xllm +pip install pre-commit +pre-commit install +git submodule update --init --recursive +``` + +执行编译，在`build/`下生成可执行文件： + +```bash +python setup.py build +``` + +编译产物路径：`build/xllm/core/server/xllm` + +## 3.启动模型 + +### 环境变量 + +```bash +# 1. 配置依赖路径相关环境变量 +export ASDOPS_LOG_TO_STDOUT=0 +export ASDOPS_LOG_LEVEL=3 +export PYTHON_INCLUDE_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])' | tail -n 1)" +export PYTHON_LIB_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])' | tail -n 1)" +export PYTORCH_NPU_INSTALL_PATH=/usr/local/libtorch_npu/ +export PYTORCH_INSTALL_PATH="$(python3 -c 'import site, os; print(os.path.join(site.getsitepackages()[0], "torch"))')" +export LIBTORCH_ROOT="$PYTORCH_INSTALL_PATH" +export LD_LIBRARY_PATH=/usr/local/libtorch_npu/lib:$LD_LIBRARY_PATH + +# 2. 加载环境 +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export ASCEND_RT_VISIBLE_DEVICES=14,15 +export ASDOPS_LOG_TO_STDOUT=1 +export ASDOPS_LOG_LEVEL=0 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export NPU_MEMORY_FRACTION=0.90 +export ATB_WORKSPACE_MEM_ALLOC_ALG_TYPE=3 +export ATB_WORKSPACE_MEM_ALLOC_GLOBAL=1 +export OMP_NUM_THREADS=12 +export HCCL_CONNECT_TIMEOUT=7200 +export INF_NAN_MODE_ENABLE=0 +export INF_NAN_MODE_FORCE_DISABLE=1 + +# 3. 清理旧日志 +LOG_DIR="log" +mkdir -p $LOG_DIR +``` + +:::note +Qwen3.5 目前不支持 TP=16 场景。 +::: + +## 启动命令 - Qwen3.5-27B（2卡 TP=2，投机解码） + +```bash +MODEL_PATH="/path/to/Qwen3.5-27B" +DRAFT_MODEL_PATH="/path/to/Qwen3.5-27B-mtp" + +MASTER_NODE_ADDR=":32764" +START_PORT=18076 +START_DEVICE=0 +NNODES=2 + +export HCCL_IF_BASE_PORT=53433 + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + ./xllm/build/xllm/core/server/xllm \ + --model $MODEL_PATH \ + --devices="npu:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --max_memory_utilization=0.7 \ + --max_tokens_per_batch=32768 \ + --max_seqs_per_batch=8 \ + --block_size=128 \ + --communication_backend="lccl" \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=true \ + --enable_schedule_overlap=true \ + --enable_graph=true \ + --node_rank=$i \ + --enable_shm=true \ + --task="generate" \ + --max_concurrent_requests=8 \ + --backend llm \ + --draft_model $DRAFT_MODEL_PATH \ + --draft_devices="npu:$DEVICE" \ + --num_speculative_tokens 3 \ + >> $LOG_FILE 2>&1 & +done +``` diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_next.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_next.md new file mode 100644 index 0000000000..5327291032 --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_next.md @@ -0,0 +1,8 @@ +--- +title: "Qwen3-Next" +description: "Qwen3-Next 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 Qwen3-Next 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_vl.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_vl.md new file mode 100644 index 0000000000..ec6b9e2047 --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_vl.md @@ -0,0 +1,8 @@ +--- +title: "Qwen3-VL" +description: "Qwen3-VL 自回归模型推理实践指南占位页" +--- + +本章节用于汇总 Qwen3-VL 自回归模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/diffusion_models.md b/docs/src/content/docs/zh/cookbook/diffusion_models.md new file mode 100644 index 0000000000..1c50256621 --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/diffusion_models.md @@ -0,0 +1,14 @@ +--- +title: "扩散模型" +description: "xLLM 扩散模型推理实践" +--- + +本章节用于汇总使用 xLLM 服务扩散模型的实践指南。 + +后续会随着实践内容整理逐步补充。 + +## 模型系列 + +- [Flux](/zh/cookbook/diffusion_models/flux/flux/) +- [Wan](/zh/cookbook/diffusion_models/wan/wan2_1/) +- [Qwen-Image](/zh/cookbook/diffusion_models/qwen_image/qwen_image/) diff --git a/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux.md b/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux.md new file mode 100644 index 0000000000..8115c4ca9e --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux.md @@ -0,0 +1,8 @@ +--- +title: "Flux" +description: "Flux 扩散模型推理实践指南占位页" +--- + +本章节用于汇总 Flux 扩散模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md b/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md new file mode 100644 index 0000000000..950945fda0 --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md @@ -0,0 +1,519 @@ +--- +title: "Flux2" +description: "Flux2 扩散模型推理实践指南占位页" +sidebar: + order: 3 +--- + +本章节用于汇总 Flux2 扩散模型在 xLLM 中的推理实践。 + ++ 源码地址：https://github.com/xLLM-AI/xllm + ++ 国内可用: https://gitcode.com/xLLM-AI/xllm + ++ 权重下载: [modelscope-FLUX.2-dev](https://www.modelscope.cn/models/black-forest-labs/FLUX.2-dev/) + +## 1.拉取镜像环境 + +首先下载xLLM提供的镜像： + +```bash +# A2 x86 +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-x86-cann9-20260605 +# A2 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-arm-cann9-20260605 +# A3 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-cann9-20260605 +``` + + +然后创建对应的容器 + +```bash +IMAGE=quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-cann9-20260605 +CONTAINER=mydocker + +docker run \ +--name $CONTAINER \ +--privileged \ +--network=host \ +--ipc=host \ +--device /dev/davinci_manager \ +--device /dev/devmm_svm \ +--device /dev/hisi_hdc \ +-v /usr/local/dcmi:/usr/local/dcmi \ +-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \ +-v /usr/local/Ascend/driver/:/usr/local/Ascend/driver/ \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /root/.cache:/root/.cache \ +-v /export/home:/export/home \ +-v /home/:/home/ \ +-w /export/home \ +-itd $IMAGE bash +docker exec -it $CONTAINER bash +``` + +## 2.拉取源码并编译 + +下载官方仓库与模块依赖： + +```bash +git clone https://github.com/xLLM-AI/xllm.git +cd xllm + +``` + +下载安装依赖: + +```bash +pip install pre-commit +pre-commit install + +git submodule update --init --recursive +``` + +执行编译，在`build/`下生成可执行文件`build/xllm/core/server/xllm`： + +```bash +python setup.py build +``` + +## 3.启动模型 + +### 若机器为重启后初次拉起服务，需先执行以下脚本对device进行初始化 + +若不执行且npu未初始化可能导致xllm进程拉起失败 + +```bash +python -c "import torch_npu +for i in range(16):torch_npu.npu.set_device(i)" +``` + +当前xLLM侧的Flux2拉起服务化是两阶段模式，需要分别拉起text-encoder组件和DiT组件服务化，再通过python的embedding脚本触发整个Flux2推理进程。另外，Flux2支持TP、SP和dit_cache特性（TaylorSeer、ResidualCache），暂时不支持chunked prefill特性。 + +### 1. 拉起text-encoder组件服务化 +#### 环境变量 + +```bash +##### 1. 配置依赖路径相关环境变量 +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +##### 2. 配置日志相关环境变量 +rm -rf core.* +export ASCEND_MODULE_LOG_LEVEL=ATB=0 +export ASDOPS_LOG_TO_FILE=1 +export ASCEND_SLOG_PRINT_TO_STDOUT=1 + +##### 3. 配置性能、通信相关环境变量 +export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 +``` + +#### 启动命令 - 启动Flux2的text-encoder组件（单机1卡2die TP=2） + +```bash +MODEL_PATH="/path/to/flux2/text_encoder/" # text_encoder路径 +MASTER_NODE_ADDR="127.0.0.1:9748" # Master 节点地址（需全局一致） +START_PORT=18001 # 服务起始端口 +START_DEVICE=2 # 起始逻辑设备号 +LOG_DIR="log" # 日志目录 +NNODES=2 # 节点数（当前脚本启动 1 个进程） + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/mistral_node_$i.log" + ./build/xllm/core/server/xllm \ + --model $MODEL_PATH \ + --devices="npu:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --max_memory_utilization=0.86 \ + --block_size=128 \ + --tp_size=2 \ + --communication_backend="hccl" \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_schedule_overlap=true \ + --enable_return_mm_full_embeddings=true \ + --enable_mistral_prompt_to_message=true \ + --task="embed" \ + --enable_shm=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` +### 2. 拉起DiT组件服务化 +#### 环境变量 + +```bash +##### 1. 配置依赖路径相关环境变量 +export PYTHON_INCLUDE_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])')" +export PYTHON_LIB_PATH="$(python3 -c 'from sysconfig import get_paths; print(get_paths()["include"])')" +export PYTORCH_NPU_INSTALL_PATH=/usr/local/libtorch_npu/ # NPU 版 PyTorch 路径 +export PYTORCH_INSTALL_PATH="$(python3 -c 'import torch, os; print(os.path.dirname(os.path.abspath(torch.__file__)))')" # PyTorch 安装路径 +export LIBTORCH_ROOT="$PYTORCH_INSTALL_PATH" # LibTorch 路径 +export LD_LIBRARY_PATH=/usr/local/libtorch_npu/lib:$LD_LIBRARY_PATH # 添加 NPU 库路径 + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +##### 2. 配置日志相关环境变量 +\rm -rf core.* +\rm -rf log/dit_node_*.log + +export ASDOPS_LOG_LEVEL=ERROR +export ASDOPS_LOG_TO_STDOUT=1 +export ASDOPS_LOG_TO_FILE=1 + +##### 3. 配置性能、通信相关环境变量 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export NPU_MEMORY_FRACTION=0.98 +export ATB_WORKSPACE_MEM_ALLOC_ALG_TYPE=3 +export ATB_WORKSPACE_MEM_ALLOC_GLOBAL=1 +export OMP_NUM_THREADS=12 +export HCCL_CONNECT_TIMEOUT=7200 +export INF_NAN_MODE_ENABLE=0 +export INF_NAN_MODE_FORCE_DISABLE=1 +export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 +``` + +#### 启动命令 - Flux2的DiT组件（单机 1卡2die TP=2） + +```bash +MASTER_NODE_ADDR="127.0.0.1:8999" # Master 节点地址（需全局一致） +START_PORT=18018 # 服务起始端口 +START_DEVICE=4 # 起始 NPU 逻辑设备号 +LOG_DIR="log" # 日志目录 +NNODES=2 # 节点数（当前脚本启动 2 个进程） + +for (( i=0; i<2; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/dit_node_$i.log" + ./build/xllm/core/server/xllm \ + --model="/path/to/flux2/" \ + --max_memory_utilization=0.6 \ + --backend="dit" \ + --tp_size=2 \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --port $PORT \ + --communication_backend="hccl" \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_schedule_overlap=false \ + --use_contiguous_input_buffer=false \ + --dit_debug_print=true \ + --enable-shm=true \ + --node_rank=$i > $LOG_FILE 2>&1 & +done +``` +### 3. 执行python的embedding脚本 + +```bash +# -*- coding: utf-8 -*- +import sys +import json +from typing import Callable, Optional, Union +from safetensors.torch import load_file +import torch +import os +import base64 +import requests +import argparse +import PIL.Image +import PIL.ImageOps +import torch +import math +import io +import numpy as np +import time + +from transformers import AutoProcessor, AutoTokenizer +CONDITION_IMAGE_SIZE = 384 * 384 +VAE_IMAGE_SIZE = 1024 * 1024 + +def load_tensor( + image: Union[str, PIL.Image.Image], + convert_method: Optional[Callable[[PIL.Image.Image], PIL.Image.Image]] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + """ + Load `image` (URL / local path / PIL.Image) and convert to torch.Tensor. + + Args: + image (str or PIL.Image.Image): URL (http/https) or filesystem path, or PIL Image. + convert_method (Callable, optional): 如果提供，会在读取后对 PIL.Image 做自定义转换并返回 PIL.Image。 + 若为 None，则会默认调用 `.convert("RGB")`。 + device (torch.device, optional): 将返回 tensor 放到哪个 device（例如 torch.device('cuda')）。 + 若为 None，则不做 device 转移（默认 CPU）。 + dtype (torch.dtype, optional): 返回 tensor 的 dtype（例如 torch.float32）。若为 None，则使用 torch.float32。 + + Returns: + torch.Tensor: shape [C, H, W], dtype float, values in [0,1], 在指定 device（若提供）。 + """ + # 读取为 PIL.Image + if isinstance(image, str): + if os.path.isfile(image): + pil_image = PIL.Image.open(image) + else: + raise ValueError( + f"Incorrect path or URL. URLs must start with `http://` or `https://`, and {image} is not a valid path." + ) + elif isinstance(image, PIL.Image.Image): + pil_image = image + else: + raise ValueError( + "Incorrect format used for the image. Should be a URL linking to an image, a local path, or a PIL image." + ) + + # 处理 EXIF 方向 + pil_image = PIL.ImageOps.exif_transpose(pil_image) + + # 自定义转换或默认 RGB + if convert_method is not None: + pil_image = convert_method(pil_image) + else: + pil_image = pil_image.convert("RGB") + + # 转 numpy 再转 tensor；确保是 contiguous 并复制内存避免引用 PIL 缓冲 + np_img = np.asarray(pil_image, dtype=np.float32) # H x W x C, float32 + # 若是灰度单通道，扩展通道 + if np_img.ndim == 2: + np_img = np_img[:, :, None] + if np_img.shape[2] == 4: + # RGBA -> RGB（简单裁掉 alpha），若需按 alpha 合成请自定义 convert_method + np_img = np_img[:, :, :3] + + # 归一化到 [0,1] + np_img = np_img / 255.0 + + tensor = torch.from_numpy(np_img).permute(2, 0, 1).contiguous() # C x H x W + + # dtype & device + target_dtype = dtype or torch.float32 + tensor = tensor.to(dtype=target_dtype) + + # clone 确保独立内存（可选，但保险） + return tensor.clone() + +def base64_to_image(base64_string, output_path): + """ + 将Base64字符串保存为图片文件 + + Args: + base64_string: Base64编码的字符串 + output_path: 输出图片路径（如：'output.jpg', 'output.png'） + """ + try: + # 解码Base64字符串 + image_data = base64.b64decode(base64_string) + + # 保存为文件 + with open(output_path, 'wb') as f: + f.write(image_data) + + print(f"图片已保存到: {output_path}") + return output_path + + except Exception as e: + print(f"转换失败: {e}") + return None + +def image_to_base64(img: Union[str, PIL.Image.Image]) -> str: + """ + 将图片文件路径或 PIL.Image 转成 Base64 字符串 + """ + if isinstance(img, str): + pil_image = PIL.Image.open(img) + elif isinstance(img, PIL.Image.Image): + pil_image = img + else: + raise ValueError("img必须是文件路径或PIL.Image对象") + + # 转RGB + pil_image = PIL.ImageOps.exif_transpose(pil_image) + pil_image = pil_image.convert("RGB") + + # 保存到内存 buffer + buf = io.BytesIO() + pil_image.save(buf, format="PNG") + byte_data = buf.getvalue() + b64_str = base64.b64encode(byte_data).decode("utf-8") + return b64_str + +def create_tensor(data, name, datatype="FP32"): + """ + 构造符合proto::Tensor格式的字典（修复后：直接对应Proto的4个顶层字段） + + Args: + data: numpy数组或Python列表，张量数据 + name: 张量名称（对应Proto的name字段） + datatype: 数据类型（对应Proto的datatype字段），默认FP32 + + Returns: + dict: 完全匹配proto::Tensor结构的字典 + """ + # 转换为numpy数组以便获取形状 + if not isinstance(data, np.ndarray): + data = np.array(data) + + # 1. 处理形状：确保为正整数（避免后端报无效维度错误） + shape = list(data.shape) + print(shape) + if any(dim <= 0 for dim in shape): + raise ValueError(f"张量{name}的形状包含非正整数：{shape}，需全部为正") + + # 2. 处理数据：展平后存入对应类型的contents字段 + contents = {} + flat_data = data.flatten().tolist() + if datatype == "FP32": + contents["fp32_contents"] = flat_data # 对应Proto的TensorContents.fp32_contents + elif datatype == "INT64": + contents["int64_contents"] = flat_data + elif datatype == "BOOL": + contents["bool_contents"] = flat_data + else: + raise ValueError(f"不支持的数据类型：{datatype}，仅支持FP32/INT64/BOOL") + + # 3. 直接返回Proto要求的4个顶层字段（无多余层级） + return { + "name": name, # 顶层name字段 + "datatype": datatype, # 顶层datatype字段（修复报错的核心） + "shape": shape, # 顶层shape字段（修复size[0]的核心） + "contents": contents # 顶层contents字段 + } + + +def test_image_generation(pos_embed): + """测试图像生成接口（使用修复后的Tensor结构）""" + api_base = "http://127.0.0.1:18018" + api_endpoint = f"{api_base}/v1/image/generation" + model_name = "flux2" + try: + # 生成示例嵌入向量（形状需符合模型要求，此处保持原逻辑） + pooled_prompt_embeds = np.random.rand(768).astype(np.float32) # 1D: [768] + prompt_embeds = np.random.rand(2, 768).astype(np.float32) # 2D: [2, 768] + + ip_adapter_image_embeds = np.random.rand(1, 4, 768).astype(np.float32) # 3D: [1,4,768] + latents = np.ones((1, 4, 32, 32), dtype=np.float32) # 4D: [1,4,32,32]（确保shape全部为正） + + # 2. 构造请求参数（Tensor结构已修复，其他逻辑不变 + payload = { + "model": model_name, + "input": { + "prompt": "A cat holding a sign that says hello world", + "prompt_2": "", + "negative_prompt": " ", + "negative_prompt_2": "", + "prompt_embed": create_tensor( + pos_embed.to(torch.float32), + name = "prompt_embeds", + datatype="FP32" + ) + }, + "parameters": { + "size": "1024*1024", + "num_inference_steps": 50, # 注意：flux-schnell推荐4步，dev推荐50步，28步可能非最优 + "guidance_scale": 2.5, # 这里需要和python侧设置一致 + "true_cfg_scale": 3.0, + "num_images_per_prompt": 1, + "seed": 42, + "max_sequence_length": 2048 + }, + "user": "test_user", + "service_request_id": f"req-{int(time.time())}" + } + print("python num_inference_steps:", 50) + # 3. 发送请求（后续逻辑不变） + headers = {"Content-Type": "application/json"} + + response = requests.post( + url=api_endpoint, + headers=headers, + data=json.dumps(payload), + timeout=60 * 5 + ) + + response.raise_for_status() + result = response.json() + # 4. 解析响应（后续逻辑不变） + print(f"接口响应: {json.dumps(result, indent=2, ensure_ascii=False)}") + # print(f"请求耗时: {time.time() - :.2f}s") + if result.get("output") and result["output"].get("results"): + for idx, image_result in enumerate(result["output"]["results"]): + print(f"\n生成图片 {idx + 1}:") + if image_result.get("url"): + print(f"URL: {image_result['url']}") + elif image_result.get("image"): + print(f"尺寸: {image_result.get('width')}x{image_result.get('height')}") + base64_to_image(image_result['image'], "./result.png") + else: + print(f"生成失败: {result.get('message', '未返回结果')}") + + except requests.exceptions.RequestException as e: + print(f"请求异常: {str(e)}") + except json.JSONDecodeError: + print("响应格式错误，无法解析为JSON") + except Exception as e: + print(f"处理失败: {str(e)}") + +def calculate_dimensions(target_area, ratio): + width = math.sqrt(target_area * ratio) + height = width / ratio + + width = round(width / 32) * 32 + height = round(height / 32) * 32 + + return width, height + +def main(args: argparse.Namespace): + start = time.time() + + tokenizer = AutoTokenizer.from_pretrained("/path/to/flux2/text_encoder/") + messages = [ + {"role": "system", "content": "You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object\nattribution and actions without speculation."}, + {"role": "user", "content": "A cat holding a sign that says hello world"}, + ] + formatted_input = tokenizer.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=False, + ) + # 4. 构造 payload，将格式化后的字符串作为 input + payload = { + "model": "text_encoder", + "input": formatted_input, + "encoding_format": "float" + } + + # 发送给mistral的请求 + raw_response = requests.post("http://127.0.0.1:18001/v1/embeddings", json=payload) + result = raw_response.json() + + # 解析向量 + bytes_data = result["data"][0]["mm_embeddings"][0]["embedding"]["contents"]["bytes_contents"] + embed_data = base64.b64decode(bytes_data) + embed_shape = result["data"][0]["mm_embeddings"][0]["embedding"]["shape"] + pos_embed = torch.frombuffer(bytearray(embed_data), dtype=torch.bfloat16).reshape(embed_shape) + + test_image_generation(pos_embed) + end = time.time() + print(f"耗时: {end - start:.2f} 秒") + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="127.0.0.1") + parser.add_argument("--port", type=int, default=18001) + parser.add_argument("--model", type=str, default="text_encoder") + parser.add_argument("--height", type=int, default=1024) + parser.add_argument("--width", type=int, default=1024) + args = parser.parse_args() + main(args) +``` \ No newline at end of file diff --git a/docs/src/content/docs/zh/cookbook/diffusion_models/qwen_image/qwen_image.md b/docs/src/content/docs/zh/cookbook/diffusion_models/qwen_image/qwen_image.md new file mode 100644 index 0000000000..e56205317f --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/diffusion_models/qwen_image/qwen_image.md @@ -0,0 +1,8 @@ +--- +title: "Qwen-Image" +description: "Qwen-Image 扩散模型推理实践指南占位页" +--- + +本章节用于汇总 Qwen-Image 扩散模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/cookbook/diffusion_models/wan/wan2_1.md b/docs/src/content/docs/zh/cookbook/diffusion_models/wan/wan2_1.md new file mode 100644 index 0000000000..c53ce2c222 --- /dev/null +++ b/docs/src/content/docs/zh/cookbook/diffusion_models/wan/wan2_1.md @@ -0,0 +1,8 @@ +--- +title: "Wan2.1" +description: "Wan2.1 扩散模型推理实践指南占位页" +--- + +本章节用于汇总 Wan2.1 扩散模型在 xLLM 中的推理实践。 + +后续会随着实践内容整理逐步补充。 diff --git a/docs/src/content/docs/zh/design/generative_recommendation_design.md b/docs/src/content/docs/zh/design/generative_recommendation_design.md new file mode 100644 index 0000000000..3c13df5293 --- /dev/null +++ b/docs/src/content/docs/zh/design/generative_recommendation_design.md @@ -0,0 +1,1080 @@ +--- +title: "生成式推荐设计文档" +sidebar: + order: 2 +--- +## 概述 + +xLLM 在 `backend=rec` 场景下提供了生成式推荐推理能力。其目标不是替代现有推荐系统，而是在保留 `predictor` 侧稀疏特征处理和在线服务能力的前提下，把 LLM 主体推理能力复用到推荐场景中，用于候选扩展、候选比较和最终结果生成。 + +本文档重点说明以下内容： + +- 生成式推荐场景的目标与约束 +- 推荐模型结构与推理接入方式 +- 为什么推荐场景更适合固定调度和整图执行 +- `xAttention` 与 `beam search` 如何围绕显存和执行效率协同优化 +- 当前分支中与生成式推荐相关的核心代码分布 + +本文档的设计目标包括： + +- 用统一视角解释 `backend=rec` 的推理链路 +- 说明固定调度、整图执行和定制算子之间的关系 +- 为后续技术分享、代码走读和文档扩展提供稳定底稿 + +本文档的非目标包括： + +- 不展开推荐模型训练细节 +- 不覆盖所有线上业务接入差异 +- 不替代各模块的详细 API 文档 + +## 1. 背景和问题 + +最近几年，基于 LLM 的生成式推荐取得了比较明显的进展。在 xLLM 中，我们也逐步补齐了对生成式推荐推理的支持。生成式推荐的目标，不是简单把大模型能力接进推荐系统，而是希望利用生成式建模能力，在候选扩展和排序阶段提升效果，尤其是提升 `CTR` 这类核心指标。 + +在当前方案中，我们使用自研 xLLM 作为统一推理引擎，通过动态库（`.so`）方式接入现有预测链路： + +- `predictor` 侧继续负责稀疏特征处理、样本组织和在线服务集成； +- `xLLM` 侧负责完成 LLM 相关推理计算。 + +这样做的价值在于，推荐系统原有的工程能力可以保留，而 xLLM 在算子、KV Cache、多后端执行和调度上的基础设施也能够直接复用。 + +但生成式推荐和通用 LLM 推理，优化目标并不相同。 + +- 通用 LLM 推理更关注逐步生成的体验，例如尽快返回第一个结果、尽量缩短每一步生成之间的间隔，并允许请求在执行过程中灵活插入和提前结束； +- 生成式推荐更关注整次请求的总时延，以及在有限几轮内得到更优的候选结果。 + +原因很直接：推荐场景通常不是生成一段开放文本，而是在固定几轮里不断扩展候选、比较候选，最后输出更优结果。 + +![生成式推荐整体背景](../../assets/generative_recommendation_overview.png) + +这里经常会用到 `beam search`。可以把它理解为：在每一轮里，不只保留当前最优的一条路径，而是同时保留多个高分候选，并在后续轮次继续扩展和比较，最后从这些候选里选出更优结果。在推荐场景里，这样做的意义不是“生成更长内容”，而是“在有限几步内覆盖更多高质量候选，提高最终推荐效果”。 + +![Beam Search 在生成式推荐中的作用](../../assets/generative_recommendation_beam_search.png) + +因此，生成式推荐天然有两个特征： + +- 固定步数推进； +- 多个候选同步比较。 + +也就是说，这个场景真正要优化的，不是“某一条序列先跑完”，而是“多个候选在固定几轮里稳定推进，并在每一轮完成低开销比较”。这也决定了后续的设计方向：调度层更适合使用固定调度，执行层更适合做整图执行，并在稳定执行形态上做专门的算子优化。 + +### 1.1 生成式推荐和通用 LLM 推理的 workload 差异 + +如果只从“模型里也有 attention”这一点来看，生成式推荐似乎和通用 LLM 推理很接近；但从服务 workload 的角度看，两者其实差异很大。 + +生成式推荐更常见的输入输出形态是： + +- 输入很长：因为用户历史行为、上下文特征、候选上下文往往都要进入模型； +- 输出很短：最终只需要生成固定长度的 item token 序列； +- decode 轮数固定：例如只生成 2 到 4 个 token 就结束； +- 每轮 decode 的代价不低：因为不是单路 greedy decode，而往往伴随着较大的 `beam_width` 和 `top_k` 候选扩展。 + +这和通用 LLM 推理正好形成一个鲜明对照： + +- 通用 LLM 推理常见的是“短 prompt + 长输出”； +- 生成式推荐常见的是“长 prompt + 短输出”。 + +这意味着生成式推荐并不会因为“输出很短”就自然变得便宜。相反，正因为输出阶段带着 `beam search`、大候选池和固定多轮比较，单步 decode 的控制和搜索成本会被放大。 + +### 1.2 这个场景最难的 3 个问题 + +从生成式推荐服务本身的 workload 特征出发，和通用 LLM 推理相比，最值得单独拎出来的有 3 类问题。 + +#### 问题一：长 prompt、短输出，但 decode 单步并不便宜 + +生成式推荐的 decode 轮数虽然固定且较少，但每一轮往往都要处理大规模候选比较。这和“输出短所以推理简单”的直觉不一样。 +如果继续沿用通用推理系统对 decode 的组织方式，就会把大量优化空间浪费在 shared prefix 的重复加载、beam 之间的冗余 KV 访问，以及 block 粒度下的重排与复制上。 + +#### 问题二：beam search 不只是搜索问题，还是系统问题 + +在生成式推荐里，`beam search` 的作用不是语言生成里的“多样性增强”，而是核心推荐候选扩展逻辑的一部分。 +一旦 `beam_width` 和 `top_k` 变大，排序、过滤、候选保活、结构复用都会成为显著成本。 +也就是说，beam search 在这里不是单独一个算法模块，而是会牵动调度、显存、数据结构和 kernel 组织方式的系统级问题。 + +#### 问题三：系统瓶颈不只在算子，也在 Host 与 Device 的协作方式 + +生成式推荐通常有严格的在线时延约束，同时并发量又高。 +如果 host 仍然按照“每一步都回来做判断、准备下一步、再发回设备”的方式控制整个过程，那么 host 调度和数据搬运本身就会成为显著瓶颈。 +这也是为什么这个场景里不仅要优化算子，还要重构整体 pipeline，让 host 和 device 的职责划分更适合 fixed-step workload。 + +### 1.3 这篇设计文档的定位 + +这篇文档不会去复述外部材料，也不试图替代论文式分析。 +更准确的定位是：它把生成式推荐场景里的工作负载特征、系统问题和当前分支的真实实现路径整理到同一份设计文档里，帮助后续做分享、走读和代码 review。 + +## 2. 推理架构 + +### 2.1 模型结构介绍 + +生成式推荐是近两年推荐系统领域的重要方向。它正在打破传统“召回-排序-重排”的级联边界，把推荐任务从“判别式匹配”推进到“生成式预测”。当前文档里重点关注两类已经在线上大规模使用的模型：用于召回的 OneRec 模型，以及用于精排的 OneTrans 模型。 + +![OneRec 模型结构](../../assets/generative_recommendation_model_onerec.png) + +![OneTrans 模型结构](../../assets/generative_recommendation_model_onetrans.png) + +从这些模型的共同点来看，它们保留了传统 CTR 场景里的序列特征、用户静态特征和上下文特征，并由输入适配层把异构推荐信号（离散 ID、连续值、序列、多模态内容）统一映射为 LLM Decoder 可理解的嵌入表示（embedding），必要时再与 LLM 的词表嵌入空间对齐。模型主体则是 LLM 的 Encoder+Decoder 或 Decoder-only 结构，因此不同部分需要不同的推理引擎承接。 + +### 2.2 推理架构介绍 + +根据模型结构特点，当前方案把模型切成两类子图： + +- 输入适配层仍然归属于传统 CTR 推理范畴，由 `predictor` 承接； +- LLM 主体部分由 xLLM 承接。 + +作为 LLM 推理的核心引擎，xLLM 在生成式推荐场景下提供了两种接入方式：RPC 接入与动态库（`.so`）接入。 + +#### 2.2.1 RPC 接入方式 + +当前营销等在线召回场景的生成式推荐主要采用 RPC 方式接入。它的优点是服务边界清晰、接入方式稳定，但也会引入额外的 RPC 调用开销。 + +#### 2.2.2 动态库接入方式 + +另一种方式是把 xLLM 作为 `predictor` 内部的独立推理引擎，对模型中属于 LLM 主体的子图直接做推理。这样可以省掉 RPC 往返开销，后续更适合承接需要低延迟的相关业务。 + +![xLLM 在生成式推荐中的接入架构](../../assets/generative_recommendation_integration_architecture.jpg) + +## 3. 固定调度与整图执行 + +### 3.1 固定步数调度 + +![Orca 中 continuous batching 的背景](../../assets/fixed_steps_scheduler_orca.png) + +上图来自论文《Orca: A Distributed Serving System for Transformer-Based Generative Models》，它介绍了 `continuous batching` 的背景：通过动态重组 batch，避免固定 batch 调度导致算力空转。 + +但生成式推荐是固定步数的，这一点改变了调度问题本身。从调度角度看，生成式推荐更适合 `fixed_steps_scheduler`，而不是 `continuous batching`。原因不只是“固定步数所以固定调度”，而是因为这个场景本身就是按固定几轮来组织计算的。既然请求通常会在约定好的几步里完成，而且多个候选需要同步向前推进，那么调度器最重要的任务就不是“随时插队、随时清退”，而是“把这一组候选稳定地发出去，并尽量减少额外调度动作”。 + +`fixed_steps_scheduler` 的第一个好处，是更适合 `beam search`。在 `decode` 阶段，`beam width` 往往比较大，我们希望多个 beam 在同一轮里一起推进、一起比较。如果采用连续调度，那么每一步都可能触发 batch 重组、sequence 压缩、索引重排和状态裁剪。这些动作在通用 LLM 推理里是合理的，因为请求确实会动态结束；但在生成式推荐里，它们很多时候并不是收益，而是额外成本。使用固定调度之后，同一个请求下的多个 beam 可以在固定窗口里齐头并进，调度器不需要每一步都重新组织 batch，也不需要反复判断哪些序列该保留、哪些序列该剔除。这样做可以明显减少调度层的控制开销。 + +第二个好处，是执行形态会更稳定。一旦解码轮数固定、beam group 规模固定、推进节奏固定，很多后续优化才真正有了基础。比如 buffer 可以提前分配，workspace 更容易复用，cache 访问模式也更规整。对于性能优化来说，这种稳定性很重要，因为它意味着更容易做 profiling、更容易做容量规划，也更容易把执行链路固化下来。换句话说，`fixed_steps_scheduler` 解决的是调度稳定性问题，它让执行入口从动态、不规则、频繁变化的状态，收敛成了一个稳定的固定窗口。 + +![PagedAttention 的问题背景与固定步场景对比](../../assets/paged_attention_comparison.png) + +第三个好处，是它减少了很多与模型计算无关的损耗。在推荐场景里，主要成本本来应该集中在真正的候选扩展、注意力计算和 beam 比较上；但如果每一步都让调度器参与 sequence 重排、batch 重组、元数据更新和索引搬运，那么会引入不少“不是算子本身、但又必须付出”的额外成本。从这个角度看，固定调度本质上是在用更强的执行确定性，换更高的吞吐、更低的调度成本以及更稳定的运行时行为。 + +当然，固定调度也有代价。最明显的问题就是，新请求的等待时间会变长。因为连续调度的一个优势，是新请求可能等一步就有机会被插入；而固定调度下，新请求通常要等当前这一轮固定窗口结束，才能进入下一轮执行。这会带来更明显的排队等待。这个问题的缓解方向，不是退回到连续调度，而是引入 `multi-stream`。也就是说，把已经在固定窗口里的大批请求和新接入的小批请求尽量解耦，让它们落在不同 stream 或不同执行通道上。这样做的目的，不是完全消除等待，而是在保住固定调度吞吐优势的同时，降低新请求接入的额外时延。 + +### 3.2 整图执行 + +在这个基础上，`multi_step_pipeline` 就成为固定调度的天然配套设计。它解决的是执行效率问题。既然我们已经知道这个场景本身就是固定几步，而且通常不会提前结束，那么就没有必要每一步都让 host 参与一次控制：没有必要每一步都做一次 `D2H` 去判断“这一批是不是结束了”，也没有必要每一步都再做一次 `H2D` 去准备下一轮输入。更高效的做法，是在第一步启动时，就把后续若干步需要用到的空间、索引和数据结构一次性准备好，然后让 device 侧连续向前推进。 + +这样做的收益非常直接： + +- 减少 `D2H/H2D` 往返，降低 host 参与频率； +- 减少每一步的 launch 和控制开销； +- 让更多中间数据停留在 device 侧，提高数据复用效率； +- 让整段执行过程更像一条连续流水，而不是“每一步停一下、准备一下、再继续”。 + +对于生成式推荐这种固定轮数任务来说，这种连续执行方式明显比逐步回到 host 再下发下一轮更高效。 + +`multi_step_pipeline` 还有一个经常被低估的价值，就是它为定制算子创造了更好的运行条件。在执行形态稳定之后，配合定制算子把关键热路径进一步做快。`fixed step` 解决的是调度稳定性，而整图执行加上算子定制，解决的是执行效率。 + +## 4. 显存管理与算子协同优化 + +### 4.1 计算与显存瓶颈 + +#### 4.1.1 模型输入输出特征 + +在当前生成式推荐推理设定中，item id 由固定长度 token 序列表示，因此 `decode_step` 是已知的小常数（例如 3）。一次请求的推理流程可以概括为： + +- 一次 prefill：输入为长序列，即用户历史上下文； +- `decode_step` 次 decode：每步生成 1 个 token，最终组合为 item id。 + +单步 decode 的单位开销并不低。为了召回与多样性，生成式推荐通常需要较大的 `beam_width`；同时每条 beam 还要扩展 `top_k` 个候选，再在全局候选池 `beam_width × top_k` 上选择新的 beam 集合，最终 beam 集合大小仍保持为 `beam_width`。例如当 `beam_width=512`、`top_k=512` 时，单步候选池大小达到 262144（约 2.6×10^5）。因此 decode 的步数虽然不多，但每步的搜索选择与 KV 访问开销仍然不低。 + +#### 4.1.2 存储冗余与显存碎片 + +生成式推荐推理服务的主要瓶颈可以拆成两类，而 `xAttention` 就是围绕这两类问题来设计的。 + +第一类是 Attention 的冗余带宽消耗：shared prefix 没有被显式建模为可复用结构。在较大的 beam 场景下，所有 beam 都共享同一段长 prompt，但通用实现往往以“每条 beam 一条完整序列”的视角组织 KV，导致 Shared KV 在 beam 维度被重复触发加载，attention kernel 的有效算术强度下降，最终受限于 HBM 带宽。 + +第二类是 KV Cache 的复制与碎片：beam 分叉与 block 级管理之间存在结构性冲突。beam search 会频繁 fork 与 retire，并触发 beam 重排。对于基于 block 的 KV 管理（例如 PagedAttention 一类），“重排 + block 对齐”往往意味着 block copy、碎片化以及额外空间浪费，显存和带宽都会被放大。 + +### 4.2 `xAttention` 设计原理 + +#### 4.2.1 KV Cache 存储优化 + +![xAttention KV cache 布局](../../assets/xattention_kv_layout.png) + +围绕当前生成式推荐推理的固定结构，xAttention 把 KV Cache 的组织方式与 attention 计算和并行策略一起重新设计，将 shared prefix 在显存层面只存一份，同时 beam 的分叉与重排不再触发高代价的数据拷贝。 + +首先，KV Cache 被按“是否共享前缀”拆成两类： + +- **Shared KV**：prefill 阶段生成的 prompt KV，所有 beam 共享同一份物理存储； +- **Unshared KV**：decode 阶段每条 beam 新生成 token 的 KV，按 token 粒度管理。 + +拆成两类 KV 之后，Unshared KV 只存储 decode 阶段产生的新 token，从而避免 block copy 与显存浪费。 + +#### 4.2.2 Attention 计算优化 + +![xAttention 三阶段执行](../../assets/xattention_three_stage_pipeline.png) + +为了避免把 Shared 与 Unshared KV 直接拼接成一个逻辑长序列，以及由此带来的访存与拷贝问题，xAttention 把一次 attention 拆成三个阶段： + +1. **shared stage**：仅对 Shared KV 计算局部 softmax 统计量与部分输出； +2. **unshared stage**：仅对 Unshared KV 计算局部统计量与部分输出； +3. **merge stage**：使用 OnlineSoftmax 把两段结果稳定合并。 + +并行化层面，会把 shared、unshared 与 merge 分配到不同执行单元和队列中形成流水线，目标是让 Shared 与 Unshared 的计算尽量重叠执行，同时把同步点压缩到最少。 + +### 4.3 Beam Search 的系统化处理 + +如果说 `xAttention` 解决的是“attention 如何更省显存、更少重复读取 Shared KV”，那么这一节解决的就是“在大候选池下，beam search 怎么避免把时间浪费在无效比较和无谓排序上”。 + +从系统视角看，beam search 在生成式推荐场景里至少包含下面几层成本： + +- 每一步要从大量候选中选出新的 beam 集合； +- 候选里并不是每个 token 组合都代表真实 item； +- 随着 decode 往前推进，旧候选会被淘汰，新候选会不断产生； +- 如果每次都新建数据结构、全量排序、全量过滤，开销会非常高。 + +因此，这一层更适合被理解成“围绕 beam search 的系统优化”，而不是单纯一个排序 kernel。 +它的目标包括： + +- 尽早终止不必要的排序； +- 在 item 空间约束下尽早过滤无效路径； +- 尽量复用已有数据结构，避免每轮反复创建和销毁候选容器。 + +对于技术分享来说，这一节最值得强调的是： +在生成式推荐里，beam search 的代价不是附属成本，而是 decode 主成本的一部分。也正因为如此，beam search 需要和 fixed-step 调度、multi-step 执行、KV cache 组织一起被系统性考虑。 + +### 4.4 多级流水线和多流并发 + +第三类问题不是某个单点算子慢，而是整个 pipeline 的分层协作方式不够适合生成式推荐。 +在当前设计里，这一节关注的是一类系统层优化思想：如何让 host、engine、worker、算子执行之间尽量重叠。 + +它至少包含 3 层含义： + +1. **Host 与 Device 的分工更清晰** + - host 尽量少做每轮控制 + - device 尽量多做连续多轮执行 + +2. **执行链路尽量流水化** + - 当前轮执行时，下一轮输入准备已经开始 + - scheduler、batch builder、worker 之间尽量减少停顿 + +3. **利用多流与多并发 pipeline 缓解等待** + - 固定窗口会带来等待代价 + - 但可以通过 `multi-stream` 或多套执行 pipeline 把这部分代价摊薄 + +因此，这一节讨论的可以理解为：把 fixed-step 场景下的调度稳定性、设备侧连续执行，以及多流并发能力组合起来的系统层设计。 + +### 4.5 为什么这条设计路线值得做 + +从生成式推荐服务的 workload 特征来看，把它单独当成一种特殊服务路径来设计调度、执行和算子，并不是“过度工程化”，而是有明确收益空间的。 + +原因在于，这个场景同时具备几种在通用 LLM 推理中不常同时出现的特征： + +- 输入长，但输出短； +- decode 轮数固定，但单轮成本高； +- `beam_width` 和候选池规模都不小； +- host 参与、batch 重排和数据搬运很容易放大整体时延。 + +所以，只要沿用通用推理路径，很多系统成本就会被保留下来；而一旦把 fixed-step、multi-step、shared/unshared KV、beam search 和多流执行放到一起设计，收益就会从多个层次同时出现：调度成本下降、host 参与变少、显存组织更合理、热路径更适合做定制算子优化。 + +所以在技术分享里，这一节可以作为一个很好的收口： +我们不是因为“fixed step 听起来合理”才这样做，而是因为生成式推荐在系统层面确实和通用 LLM 推理不一样，这样做能换来更高吞吐和更稳定的低时延表现。 + +## 5. 代码结构 + +当前分支里，生成式推荐相关代码可以按下面的结构理解： + +外部接入： +- `xllm/c_api/rec.h` +- `xllm/c_api/internal/rec.cpp` +- `xllm/c_api/examples/simple_rec_completions.cpp` + +服务入口： +- `xllm/api_service/rec_completion_service_impl.cpp` +- `xllm/api_service/chat_service_impl.cpp` +- `xllm/api_service/api_service.cpp` +- `xllm/api_service/api_service.h` + +调度与引擎： +- `xllm/core/distributed_runtime/rec_master.cpp` +- `xllm/core/distributed_runtime/rec_master.h` +- `xllm/core/scheduler/fixed_steps_scheduler.cpp` +- `xllm/core/scheduler/fixed_steps_scheduler.h` +- `xllm/core/distributed_runtime/rec_engine.cpp` +- `xllm/core/distributed_runtime/rec_engine.h` + +batch / request / proto： +- `xllm/core/framework/batch/rec_batch_input_builder.cpp` +- `xllm/core/framework/batch/rec_batch_input_builder.h` +- `xllm/core/framework/batch/rec_multi_round_batch_input_builder.cpp` +- `xllm/core/framework/batch/rec_multi_round_batch_input_builder.h` +- `xllm/core/framework/request/rec_type.h` +- `xllm/proto/rec.proto` +- `xllm/proto/completion.proto` +- `xllm/proto/xllm_service.proto` + +runtime / worker： +- `xllm/core/runtime/rec_worker_impl.cpp` +- `xllm/core/runtime/rec_worker_impl.h` + +kernel / 算子热路径： +- `xllm/core/layers/cuda/xattention.cpp` +- `xllm/core/layers/cuda/flashinfer_attention.cpp` +- `xllm/core/kernels/cuda/xattention/beam_search.cpp` +- `xllm/core/kernels/cuda/xattention/cache_select.cu` + +## 6. 当前分支的执行主链 + +为了把设计和实现真正对起来，可以把当前分支的主执行链拆成下面几步来理解： + +1. **外部接入** + - 如果走动态库方式，请求会从 `xllm/c_api/internal/rec.cpp` 中的 `xllm_rec_text_completions`、`xllm_rec_token_completions` 或 `xllm_rec_chat_completions` 进入。 + - 如果走服务方式，请求会从 `xllm/api_service/rec_completion_service_impl.cpp` 或 `chat_service_impl.cpp` 进入，再转到 `RecMaster`。 + +2. **请求进入 `RecMaster`** + - `RecMaster` 负责把 prompt、token ids、raw embedding 等不同入口统一收敛到 request 构造逻辑。 + - 在这里会根据模型类型区分 `kOneRec` 和 `kLlmRec`，并选择不同的 request pipeline。 + +3. **进入固定调度** + - `RecMaster` 在初始化时直接创建 `FixedStepsScheduler`。 + - 调度器不再按“每一步都动态重排 batch”的思路工作，而是优先围绕固定轮数和固定候选组去构造 batch。 + +4. **引擎执行** + - `RecEngine` 再根据 `RecPipelineType` 选择执行路径。 + - 对 `LlmRec` multi-round 场景，会下沉到 `RecMultiRoundEnginePipeline`，把多轮 decode 的主要控制逻辑继续往 worker 侧下压。 + +5. **batch 与输入拼装** + - `RecBatchInputBuilder` 和 `RecMultiRoundBatchInputBuilder` 负责把 sequence、step 信息、decode positions、sampling params 等整理成 `ForwardInput`。 + - 这里的 `step_meta` 是 multi-step 执行的关键数据来源，它决定后续每一轮 decode 该如何构造位置、cache 和 beam 相关输入。 + +6. **worker 侧多轮执行** + - `RecWorkerImpl::LlmRecMultiRoundPipeline::step()` 会在设备侧循环多轮。 + - 它会先准备 beam search tensor、full/unshared KV 相关结构，再在每一轮中执行： + - 当前轮输入准备 + - 模型 forward + - sample 输出处理 + - beam search + - cache select + - 下一轮输入预计算 + +7. **算子热路径** + - Attention 相关路径落在 `xattention.cpp` 与 `flashinfer_attention.cpp` + - beam 相关路径落在 `beam_search.cpp` + - beam 重排后的 cache 选择路径落在 `cache_select.cu` + +如果从技术分享视角来讲，这条主链非常适合作为“架构总图”之后的第一条展开线，因为它把“固定调度、整图执行、定制算子”三件事串成了一个具体执行过程，而不是三个割裂的优化点。 + +## 7. 设计取舍与适用边界 + +这套设计并不意味着固定调度一定优于连续调度，也不意味着 multi-step pipeline 适合所有生成任务。它成立的前提，是当前生成式推荐场景具有下面几个特征： + +- decode 轮数较固定，通常不会像开放式文本生成那样提前结束； +- 同一请求下存在较大的 `beam_width`，而且多个 beam 需要同步比较； +- 整次请求的总时延，比逐 token 的交互体验更重要； +- 设备侧状态（KV Cache、positions、beam tensors）可以提前组织并稳定复用。 + +在这些前提成立时，固定调度和整图执行的收益会比较明显。但它也有明确边界： + +### 7.1 固定调度的边界 + +- 如果请求长度差异极大，而且大量请求会提前结束，那么连续调度的灵活性会更有价值； +- 如果业务更关心“新请求能不能立刻插入”，而不是“当前窗口吞吐是否最优”，固定调度会天然吃亏； +- 如果候选扩展不依赖大规模 beam 同步推进，那么固定窗口的收益会下降。 + +### 7.2 multi-step pipeline 的边界 + +- 如果每一步都必须回到 host 做强控制决策，那么 multi-step pipeline 的优势会被削弱； +- 如果 shape、batch 或关键输入在每一轮都大幅波动，那么想要把多轮执行稳定下来会更难； +- 如果后端算子本身还不支持稳定的多轮设备侧推进，那么整图执行只会停留在概念层。 + +### 7.3 定制算子的边界 + +`xAttention` 和 `beam search` 定制算子之所以值得做，是因为当前执行形态已经足够稳定。如果没有 fixed-step 带来的稳定 batch 形态，也没有 multi-step pipeline 带来的稳定多轮推进，那么很多定制优化都会被反复的数据搬运、batch 重组和 host 参与开销抵消掉。 + +因此，更合理的理解顺序不是“先有定制算子，再决定调度”，而是： + +1. 先确认 workload 适合固定调度； +2. 再确认多轮执行可以尽可能下沉到设备侧； +3. 最后再围绕真正稳定下来的热路径做算子定制。 + +这样才能让 `fixed_steps_scheduler`、`multi_step_pipeline`、`xAttention` 和 `beam search` 四者形成一套前后自洽的设计，而不是彼此孤立的优化点。 + +## 8. 代码路径附录 + +这一节的目的，不是重复“代码结构”里的文件清单，而是给读者一个更可执行的阅读顺序：如果后续要继续做技术分享、走读代码，或者排查 `backend=rec` 路径上的行为差异，可以直接按下面的顺序进入。 + +### 8.1 从外部入口开始看 + +如果要理解 `predictor` 或动态库接入是怎样进入 xLLM 的，建议先看下面几处： + +- `xllm/c_api/rec.h` + - 对外暴露 `xllm_rec_create`、`xllm_rec_initialize`、`xllm_rec_text_completions`、`xllm_rec_token_completions`、`xllm_rec_chat_completions` + - 适合先理解“外部系统到底能怎么调用 REC 能力” +- `xllm/c_api/internal/rec.cpp` + - 这是真正的 CAPI 实现 + - 适合看 `.so` 模式下，request 参数是怎样被封装和转发的 +- `xllm/c_api/examples/simple_rec_completions.cpp` + - 是最短的调用示例 + - 如果要给新人解释“动态库接入长什么样”，这里最直观 + +如果技术分享里想给一段“最小调用样例”，这一层最适合出现在开头。 + +### 8.2 从服务入口看统一分发 + +如果你更关心 RPC 或统一服务链路，可以继续看： + +- `xllm/api_service/api_service.cpp` + - 这里会根据 `FLAGS_backend` 决定挂哪类 service impl + - `backend == "rec"` 时，`rec_completion_service_impl_` 和 `chat_service_impl_` 都会被接上 +- `xllm/api_service/rec_completion_service_impl.cpp` + - 负责把 rec completion 请求转给 `RecMaster` + - `routing`、`input_tensors`、`RequestParams` 都是在这里被整理进来 +- `xllm/api_service/chat_service_impl.cpp` + - 对 `RecMaster` 也有 chat 入口 + - 适合说明“REC 不是只有 token completion，一样可以走 chat 形态” + +这一层适合在技术分享中回答一个问题：为什么说 `backend=rec` 不是另起炉灶，而是接进了现有服务框架。 + +### 8.3 从调度和引擎看主链 + +如果分享的重点是“为什么 fixed step 更适合 rec”，阅读顺序建议是： + +1. `xllm/core/distributed_runtime/rec_master.h` +2. `xllm/core/distributed_runtime/rec_master.cpp` +3. `xllm/core/scheduler/fixed_steps_scheduler.h` +4. `xllm/core/scheduler/fixed_steps_scheduler.cpp` +5. `xllm/core/distributed_runtime/rec_engine.h` +6. `xllm/core/distributed_runtime/rec_engine.cpp` + +可以按下面这条链理解： + +```text +Rec request + -> RecMaster + -> FixedStepsScheduler + -> RecEngine + -> RecEnginePipeline + -> Worker / worker_clients +``` + +这里最值得讲的几个点是： + +- `RecMaster` 负责入口收敛和 pipeline 选择 +- `FixedStepsScheduler` 负责把 request 组织成适合固定轮数推进的 batch +- `RecEngine` 负责把调度结果交给实际执行路径 +- `RecMultiRoundEnginePipeline` 代表“多轮 decode 控制进一步下沉”的实现方式 + +如果分享时要强调“fixed step 不是口头概念，而是代码主链的真实选择”，这一层就是核心证据。 + +### 8.4 从 batch builder 看 multi-step 的输入组织 + +如果要解释 `multi_step_pipeline` 为什么能成立，单看 scheduler 还不够，必须继续看 batch builder： + +- `xllm/core/framework/batch/rec_batch_input_builder.h` +- `xllm/core/framework/batch/rec_batch_input_builder.cpp` +- `xllm/core/framework/batch/rec_multi_round_batch_input_builder.h` +- `xllm/core/framework/batch/rec_multi_round_batch_input_builder.cpp` +- `xllm/core/framework/batch/batch.cpp` + +这里最重要的不是“类名”，而是几个关键事实： + +- `RecBatchInputBuilder::create(...)` 会按 `RecType` 和 multi-round 模式选择 builder +- `RecMultiRoundBatchInputBuilder` 不是普通 builder 的轻微变种，而是专门为多轮 decode 组织输入的实现 +- `step_meta`、`decode_positions`、`sampling params`、`batch forward type` 等信息是在这一层被拼好并送往后续 runtime 的 + +所以，如果要说明“为什么第一步就能把后面几步的输入准备好”，这一层比只讲 engine 更关键。 + +### 8.5 从 worker 看 device 侧多轮执行 + +`multi_step_pipeline` 真正最值得展开的代码在： + +- `xllm/core/runtime/rec_worker_impl.h` +- `xllm/core/runtime/rec_worker_impl.cpp` + +尤其是下面这些点： + +- `RecWorkerImpl::step_async(...)` + - 说明请求是如何进到 worker 内部并在特定 stream 上执行的 +- `RecWorkerImpl::LlmRecMultiRoundPipeline::prepare_inputs(...)` + - 说明 multi-round 输入如何进入 runtime +- `RecWorkerImpl::LlmRecMultiRoundPipeline::allocate_kv_caches_related()` + - 说明为什么 fixed-step 场景更适合提前分配 KV 相关结构 +- `RecWorkerImpl::LlmRecMultiRoundPipeline::step(...)` + - 这是最核心的一段 + - 明确展示了多轮循环、beam search、cache select、下一轮输入预计算之间的关系 +- `compute_next_round_input_async(...)` + - 这是解释“为什么可以减少 host 往返”的关键点 + +如果技术分享想从“执行效率”而不是“调度策略”切入，这一层是最值得重点展开的。 + +### 8.6 从 kernel 热路径看为什么定制算子值得做 + +如果要讲 `xAttention` 和 `beam search` 定制算子，推荐按下面的顺序进： + +- `xllm/core/layers/cuda/xattention.cpp` +- `xllm/core/layers/cuda/flashinfer_attention.cpp` +- `xllm/core/kernels/cuda/xattention/xattention_ops_api.h` +- `xllm/core/kernels/cuda/xattention/beam_search.cpp` +- `xllm/core/kernels/cuda/xattention/cache_select.cu` + +这一层适合回答 3 个问题： + +1. Attention 的稳定执行路径到底落在哪 +2. Beam Search 为什么不只是调度问题，而是 kernel 热路径问题 +3. beam 重排之后的 cache select 为什么必须和前面的执行形态一起考虑 + +也就是说，技术分享如果要把 `fixed_steps_scheduler`、`multi_step_pipeline`、`xAttention` 和 `beam search` 串成一条线，最终一定会落到这里。 + +### 8.7 推荐的代码阅读顺序 + +如果后续你或者其他同学还要继续扩写这篇文档，我建议代码阅读顺序固定成下面这样： + +```text +入口 + -> api_service + -> RecMaster + -> FixedStepsScheduler + -> RecEngine + -> RecBatchInputBuilder / RecMultiRoundBatchInputBuilder + -> RecWorkerImpl::LlmRecMultiRoundPipeline + -> xAttention / beam_search / cache_select +``` + +这个顺序的好处是： + +- 先看“请求怎么进来” +- 再看“为什么 fixed step” +- 再看“multi-step 是怎么在设备侧成立的” +- 最后看“定制算子为什么在这里有价值” + +这样逻辑最顺，也最适合技术分享展开。 + +## 9. 关键代码锚点索引 + +如果后续需要继续写技术分享、补代码注释，或者在评审时快速证明文档里的说法来自当前分支实现，可以直接从下面这组锚点入手。 + +### 9.1 入口与服务层锚点 + +- `xllm/core/distributed_runtime/rec_master.cpp:575` + - `RecMaster::handle_request(...)` + - 对应 prompt / prompt_tokens / input_tensors 入口 +- `xllm/core/distributed_runtime/rec_master.cpp:603` + - `RecMaster::handle_request(...)` + - 对应 chat messages 入口 +- `xllm/core/distributed_runtime/rec_master.cpp:651` + - `RecMaster::handle_request(const std::vector& prompt_tokens, ...)` + - 对应 token / raw input 类入口 + +这些锚点适合回答一个问题：REC 请求到底是怎么被收敛到 `RecMaster` 里的。 + +### 9.2 调度层锚点 + +- `xllm/core/scheduler/fixed_steps_scheduler.cpp:337` + - `FixedStepsScheduler::step(const absl::Duration& timeout)` + - 这是 fixed-step 调度真正往前推进的一步 +- `xllm/core/scheduler/fixed_steps_scheduler.cpp:186` + - `FixedStepsScheduler::prepare_batch()` + - 适合解释当前 batch 是怎么在固定步场景下被组织的 +- `xllm/core/framework/batch/rec_batch_input_builder.cpp:29` + - `RecBatchInputBuilder::create(...)` + - 适合解释 builder 是如何按 `RecType` 和 multi-round 模式切换的 + +这几处放在一起，可以直接支持“fixed_steps_scheduler 不是概念，而是代码主链里的真实选择”这一点。 + +### 9.3 引擎与多轮执行锚点 + +- `xllm/core/distributed_runtime/rec_engine.cpp:901` + - `RecEngine::RecMultiRoundEnginePipeline::step(...)` + - 说明 engine 层如何把执行进一步下沉到 multi-round pipeline +- `xllm/core/runtime/rec_worker_impl.cpp:849` + - `RecWorkerImpl::LlmRecMultiRoundPipeline::step(...)` + - 这是最关键的一段，真正体现多轮 decode 循环在设备侧发生 +- `xllm/core/runtime/rec_worker_impl.cpp:1011` + - `xllm::kernel::cuda::beam_search(...)` 调用点 +- `xllm/core/runtime/rec_worker_impl.cpp:1066` + - `xllm::kernel::cuda::cache_select(...)` 调用点 + +如果你需要在分享中明确“beam search 和 cache select 不是调度层概念，而是直接落到 device 热路径上的实现”，这一组锚点最适合引用。 + +### 9.4 这组锚点怎么用 + +这组锚点不需要在正文中全部展开，但非常适合作为： + +- 技术分享讲稿里的“代码证据页” +- PR 说明里的“关键实现位置” +- 后续 reviewer 问“这句话代码在哪”的快速答复 + +如果后续还要继续扩写文档，建议优先围绕这几处函数补更细的输入输出说明，而不是再扩展泛化描述。 + +## 10. 推荐讲解顺序与阅读策略 + +前面的“代码路径附录”和“关键代码锚点索引”更像是资料库，适合在需要时回查；但如果这篇文档要真正变成一场技术分享的底稿，还需要一条更偏“讲述顺序”的线索。下面给出一个更适合对外讲的展开顺序。 + +### 10.1 推荐的讲解顺序 + +如果听众并不直接参与 `backend=rec` 的实现，建议不要一上来就讲 `RecWorkerImpl` 或 `beam_search.cpp`，而是按下面的顺序推进： + +1. **先讲业务目标** + - 为什么生成式推荐和通用 LLM 推理的目标不一样 + - 为什么这里更关注“整次请求的总时延”和“固定几轮的候选比较” + +2. **再讲调度选择** + - 为什么 `fixed_steps_scheduler` 更适合这个 workload + - 为什么大 `beam_width` 会让固定调度比连续调度更划算 + +3. **然后讲执行方式** + - `multi_step_pipeline` 为什么可以把多轮 decode 控制下沉到设备侧 + - 为什么这样可以减少 host 往返和控制开销 + +4. **再讲定制算子** + - 为什么执行形态稳定之后，`xAttention` 和 `beam search` 优化才真正值得做 + - 为什么这两类优化要放在一节里讲，而不是拆成孤立话题 + +5. **最后回到代码** + - 再用代码主链和锚点证明前面的结论，不让整场分享停留在概念层 + +这个顺序的好处是：听众先理解“为什么”，再理解“怎么做”，最后再看“代码在哪里”。这比一开始就从实现文件名切入更容易跟上。 + +### 10.2 如果是内部走读，顺序可以反过来 + +如果听众本身就是 xLLM 或推荐基础设施相关同学，那么也可以采用另一条顺序： + +1. 先看 `RecMaster -> FixedStepsScheduler -> RecEngine` +2. 再看 `RecBatchInputBuilder` +3. 再看 `RecWorkerImpl::LlmRecMultiRoundPipeline` +4. 最后看 `xAttention / beam_search / cache_select` + +这种讲法更适合代码走读，因为它直接沿着调用栈往下走。但它的缺点是，对没有上下文的听众来说，一开始就会掉进实现细节里，不容易先抓住设计选择背后的动机。 + +### 10.3 建议在分享中强调的 3 个结论 + +如果后续要把这篇文档压缩成 10 分钟技术分享，我建议把整篇内容收束成下面 3 句话： + +- `fixed_steps_scheduler` 解决的是调度稳定性问题； +- `multi_step_pipeline` 解决的是多轮执行效率问题； +- `xAttention` 和 `beam search` 定制算子是在执行形态稳定之后，进一步兑现性能收益的关键路径优化。 + +这三句话是整篇文档最值得被记住的部分，后面的代码链路和锚点都可以理解为是在为这三句结论提供证据。 + +## 11. 与通用 LLM 推理路径的对照 + +为了避免把 `backend=rec` 看成“只是把 LLM 推理拿来改一改”，这里把它和通用 LLM 推理的主路径做一个更明确的对照。 + +### 11.1 优化目标不同 + +通用 LLM 推理更强调逐 token 的生成体验，例如尽快返回第一个结果、尽量缩短 token 间隔，并让新的请求尽快插入执行。 +而 `backend=rec` 更强调固定轮数内的候选扩展与比较，因此更关注整次请求的总时延，而不是单条序列的最早结束时间。 + +这意味着两者虽然都在做 decode，但优化目标已经发生了偏移： + +- 通用 LLM 推理更偏“动态请求管理” +- 生成式推荐更偏“固定窗口内的同步推进” + +### 11.2 调度重点不同 + +在通用 LLM 推理里，连续调度的价值来自于： + +- 某些序列提前结束后可以立刻让位； +- batch 可以被持续重组； +- decode 路径中动态插入和动态退出是高频动作。 + +而在 `backend=rec` 里： + +- decode 轮数更固定； +- `beam_width` 更大； +- 同一个 request 的多个候选需要在同一步比较； +- 频繁重排 batch 反而会放大调度成本。 + +所以，通用 LLM 推理的调度器重点是“灵活”，而生成式推荐的调度器重点是“稳定”。 + +### 11.3 执行方式不同 + +通用 LLM 推理常常允许 host 在每一步都继续参与调度、结束判断和下一轮输入准备。 +而生成式推荐因为轮数固定、候选同步推进，所以更适合在第一步时就把后续几步所需的结构准备好，再让 device 侧连续推进。 + +这也是为什么 `multi_step_pipeline` 对推荐场景更有价值：它不是简单减少几个 memcpy，而是把原本“每一步都回来问一下 host”的控制模式，替换成“尽量在设备侧连续完成多轮”的模式。 + +### 11.4 算子收益兑现方式不同 + +通用 LLM 推理里，算子优化往往直接服务于单步 decode 或通用 attention 路径。 +而在 `backend=rec` 中，算子优化的价值高度依赖执行形态是否已经稳定下来。 + +如果固定调度没有成立，batch 还在频繁重组；如果 multi-step pipeline 没有成立，host 还在反复介入，那么很多算子级优化都会被额外的数据搬运和控制开销抵消掉。 +因此，在推荐场景里更合理的顺序是： + +1. 先把调度稳定下来； +2. 再把多轮执行尽量下沉到设备侧； +3. 最后才围绕真正稳定的热路径做 `xAttention`、`beam search`、`cache_select` 这类优化。 + +### 11.5 设计边界与适用条件 + +上面的对照并不是为了强调“生成式推荐和通用 LLM 推理完全不同”，而是为了明确哪些设计可以直接复用通用推理框架，哪些设计必须围绕 `backend=rec` 的 workload 特征单独处理。 + +从当前分析看，`fixed_steps_scheduler`、`multi_step_pipeline`、`xAttention` 和 `beam search` 这四部分之所以需要被放在一起讨论，本质原因是它们依赖同一组前提条件： + +- 输出长度固定或近似固定； +- decode 轮数较少，但单轮计算成本高； +- 同一请求下需要维护较大的 `beam_width`； +- 候选扩展和候选比较是主路径，而不是附属逻辑； +- 设备侧状态可以提前组织，并在后续轮次中稳定复用。 + +只有在这些条件同时成立时，固定调度和整图执行的收益才会比较稳定。如果离开这些条件，例如请求长度差异极大、序列可能随时提前结束、或者新请求接入时延比吞吐更重要，那么 `backend=rec` 当前这套设计就不一定仍然是最优选择。 + +进一步看，这四部分之间并不是并列关系，而是逐层依赖关系。 + +第一层是 `fixed_steps_scheduler`。它解决的是调度入口的稳定性问题。如果调度层仍然每一步都在重组 batch、裁剪 sequence、回收空位，那么后面执行层的稳定性基础就不存在。 + +第二层是 `multi_step_pipeline`。它解决的是执行过程的连续性问题。只有在调度窗口已经稳定之后，才有意义进一步把多轮 decode 的输入准备、KV 组织和轮次推进尽量下沉到 device 侧。 + +第三层是 `xAttention`。它解决的是 attention 在生成式推荐场景下的显存与访存问题。只有在执行过程已经相对稳定之后，shared KV 与 unshared KV 的拆分、分段 attention 和流水线计算才会稳定兑现收益。 + +第四层是 `beam search` 优化。它解决的是大候选池下的搜索与筛选成本问题。由于 beam 相关逻辑会直接影响 decode 阶段的主路径，因此它必须和前面三层设计一起考虑，而不能独立看成一个后处理模块。 + +因此，这里真正想说明的是：当前分支中的这套设计不是四个离散优化点，而是一套围绕生成式推荐 workload 逐层收敛出来的组合方案。调度先稳定，执行再连续，算子最后吃收益。如果顺序反过来，很多优化很可能在系统层被额外开销抵消。 + +### 11.6 不适用或收益有限的场景 + +为了避免把当前方案理解成“生成式推荐的统一答案”，还需要明确它的边界。 + +第一类是不适合固定调度的场景。比如请求长度波动极大、序列经常提前结束、或者系统更关心新请求的最短接入延迟而不是整批吞吐。在这种情况下，连续调度的灵活性可能更重要，固定窗口反而会放大等待成本。 + +第二类是不适合整图执行的场景。比如每一轮都必须由 host 做强控制决策，或者每一轮的 shape、batch 结构、关键输入变化都非常剧烈。这类场景下，`multi_step_pipeline` 很难把多轮控制稳定地下沉到 device 侧，整图执行也就不容易获得稳定收益。 + +第三类是算子侧收益不明显的场景。如果 beam 不大、候选扩展成本不高、或者 shared prefix 不长，那么 `xAttention` 和 `beam search` 相关优化虽然依然成立，但其工程收益可能不会像典型生成式推荐场景那样突出。 + +从技术文档角度，把这些边界写清楚有两个好处：一是避免读者误以为这套方案无条件普适；二是让后续 reviewer 在看实现时能更明确地区分“设计前提”与“实现缺陷”。 + +## 12. 验证与验收建议 + +这一节的验证不应只停留在“文档能不能打开”，而应该和前文所描述的设计目标直接对应。换句话说，验证要回答的问题是：当前分支中的实现，是否真的支持文档里所描述的这套设计。 + +### 12.1 文档层验证 + +最基础的验证仍然需要保留： + +- 中文与英文设计文档能够正常渲染； +- 所有图片引用都能正确解析到 `docs/assets/`； +- 英文文档引用的是英文版示意图，而不是中文标注图片； +- 入口页和相关 feature 文档能够导航到这篇设计文档。 + +这一层验证的意义，是保证文档本身可以被正常消费。 + +### 12.2 代码路径一致性验证 + +第二层验证，是确认文档中提到的关键路径在当前分支中确实存在，并且角色描述与实际实现一致。这里建议至少对以下路径做静态核对： + +- `RecMaster` 是否确实承担 request 收敛与 pipeline 选择； +- `FixedStepsScheduler` 是否确实是 `backend=rec` 的固定调度入口； +- `RecEngine` 是否承担 engine 层执行组织； +- `RecWorkerImpl::LlmRecMultiRoundPipeline` 是否确实承担多轮 device 侧执行； +- `xAttention`、`beam_search`、`cache_select` 的 kernel 路径是否与文档描述一致。 + +这一层验证的目的，是避免文档讲的是“理想结构”，而代码里其实已经偏离。 + +### 12.3 设计目标对应的验证项 + +如果要进一步把这篇文档作为技术分享和实现说明的统一底稿，建议把验证项和设计目标一一对应起来。 + +针对固定调度，建议关注： + +- decode 过程中 batch 是否仍频繁重组； +- beam 相关 sequence 是否仍在每轮发生大量裁剪和搬移； +- 调度开销是否在 profile 中被压低。 + +针对 `multi_step_pipeline`，建议关注： + +- host 是否仍在每一轮都做终止判断； +- 下一轮输入是否已经可以在当前轮计算期间准备； +- `D2H/H2D` 往返是否相比逐轮控制路径有所下降。 + +针对 `xAttention`，建议关注： + +- shared KV 是否真的只保留一份物理存储； +- unshared KV 是否只承载 decode 产生的新 token； +- 是否减少了 shared prefix 的重复加载。 + +针对 `beam search`，建议关注： + +- 大 `beam_width` 场景下排序和过滤是否仍然占主成本； +- 相关数据结构是否可以跨轮复用； +- item 过滤和候选筛选是否已经与 decode 主链有效衔接。 + +### 12.4 最终验收标准 + +如果把这篇文档作为当前分支生成式推荐设计的验收基线，那么最终至少应满足以下标准： + +- 文档层面：中英文可读、引用完整、导航可达； +- 结构层面：调度、执行、算子三层关系讲清楚； +- 代码层面：关键路径、关键函数、关键文件能对上； +- 设计层面：适用条件、边界条件和主要取舍写清楚； +- 分享层面：读者能够从文档中直接抽出讲稿主线，而不需要再次重构逻辑。 + +只有同时满足这几类条件，这篇文档才不只是“说明写过了”，而是真正能作为设计、分享、review 三者共用的稳定底稿。 + +## 13. FAQ / 常见误解 + +### Q1: 固定步调度是不是一定比连续调度更好？ + +不是。固定步调度之所以在 `backend=rec` 场景更合适，是因为这里的 decode 轮数更固定、`beam_width` 更大、候选比较更同步。如果这些前提不成立，连续调度的灵活性可能更有价值。 + +### Q2: `multi_step_pipeline` 是不是只是少做几次 memcpy？ + +不是。减少 `D2H/H2D` 往返只是表面收益。更关键的是，它改变了控制方式：从“每一步都依赖 host 回来做决策”，变成“尽量在 device 侧连续推进多轮执行”。 + +### Q3: `xAttention` 和 `beam search` 为什么要放在同一节？ + +因为它们都依赖同一件事：执行形态已经稳定。如果调度还在频繁重组 batch、host 还在每一步强介入，那么不管是 attention 优化还是 beam search 优化，都会被系统层额外开销稀释。 + +### Q4: 这套设计是不是只适用于 OneRec？ + +不是。当前文档以生成式推荐场景为主线，重点落在 `backend=rec` 的固定步和多轮执行特征上。OneRec 是重要例子，但重点不是某个具体模型名，而是这类 workload 的服务特征。 + +### Q5: 为什么文档里反复强调“先稳定调度，再优化算子”？ + +因为在系统设计里，顺序很重要。如果调度入口不稳定，batch 形态不稳定，执行过程也不能稳定地下沉到设备侧，那么算子级优化很难持续兑现收益。只有调度和执行先稳定，算子优化才不会被反复的数据搬运和控制开销抵消。 + +## 14. 接口与数据契约 + +如果这篇文档要继续作为实现和评审的基础材料，仅靠“原理解释”还不够，还需要把一些关键接口和数据契约明确写出来。这里不追求覆盖所有字段，而是把最影响实现正确性的部分固定下来。 + +### 14.1 外部入口的输入形态 + +当前 `backend=rec` 至少存在三类外部输入形态： + +- **文本 prompt 入口** + - 适用于 prompt 驱动的推荐请求 + - 最终会被收敛到 `RecMaster::handle_request(...)` +- **token / raw 输入入口** + - 适用于已经完成 tokenizer 处理或外部直接传 token 序列的场景 + - 也可能带有 embedding 或索引信息 +- **chat 入口** + - 适用于对话式 recommendation 场景 + - 在服务层最终仍然会落回 `RecMaster` + +从设计角度看，这里最重要的并不是“有多少入口”，而是：所有入口最终都要收敛到统一 request state 和统一 scheduler / engine / worker 主链中。 + +### 14.2 与 multi-round 直接相关的关键数据 + +对 `backend=rec` 来说，下面几类数据是关键契约： + +- `beam_width` + - 决定每轮保留多少候选分支 +- `top_k` + - 决定每轮每个分支扩展多少候选 +- `decode_step` / `total_round` + - 决定固定轮数和整图执行边界 +- `decode_positions` + - 决定每一轮 token 的位置组织方式 +- shared / unshared KV + - 决定 decode 阶段 KV cache 如何拆分和复用 + +如果这些数据没有被提前组织好，那么 fixed-step 和 multi-step 的收益就无法稳定落地。 + +### 14.3 文档层应该长期保持的不变量 + +后续如果继续扩写这篇文档，我建议至少保持下面几条不变量： + +- 文档里一旦提到 fixed-step，就必须同时交代 fixed-step 的边界条件； +- 文档里一旦提到 multi-step，就必须说明 host 侧减少了哪些职责； +- 文档里一旦提到 `xAttention`，就必须解释它和 KV 组织之间的关系； +- 文档里一旦提到 `beam search`，就不能只讲算法，还要讲它的系统成本； +- 文档里的代码锚点必须能在当前分支对上，不能引用未来规划路径。 + +## 15. 实现对齐表 + +为了避免读者把“设计意图”和“当前分支已经落地的能力”混在一起，这里给出一个简化的实现对齐表。它不是精确到每个 helper 的状态表，而是当前文档里最重要的设计项对齐结果。 + +### 15.1 已有实现 + +- **固定调度** + - 当前分支已有 `FixedStepsScheduler` + - 可以作为固定窗口调度的代码事实依据 +- **multi-round 执行主链** + - 当前分支已有 `RecMultiRoundEnginePipeline` + - worker 侧已有 `LlmRecMultiRoundPipeline` +- **beam search 热路径** + - 当前分支已有 beam search 相关调用与 kernel 路径 +- **cache select 热路径** + - 当前分支已有 cache select 调用和对应 CUDA 路径 +- **xAttention / flashinfer 路径** + - 当前分支已有对应实现文件和调用链 + +### 15.2 当前文档里仍然属于“设计抽象”的部分 + +- `xAttention`、beam search、fixed-step、multi-step 之间的收益关系 + - 当前文档做了归纳 + - 但它们不是某个单一类或单一模块的名字 +- 多流缓解等待的整体收益 + - 当前代码里有并发 pipeline 与 stream 的基础设施 + - 但文档里的收益分析仍属于系统层抽象，不是单个类能直接表达的事实 + +### 15.3 为什么实现对齐表值得单独写 + +这一节的作用，是避免 reviewer 或听众把“文档里讲得通”误以为“当前分支里一定已经逐项完全实现”。 +实现对齐表能让后续讨论更聚焦：哪些内容是已经可以拿代码证明的，哪些内容是当前设计的系统层解释。 + +## 16. 失败模式与可观测性 + +如果这篇文档只讲“正常路径”，那它仍然更像分享稿，而不是技术文档。在线系统一定要考虑失败模式与观测手段，因此这里补一个最小版本。 + +### 16.1 典型失败模式 + +- **等待时间变长** + - fixed-step 窗口导致新请求排队时间增加 +- **beam 过大导致 decode 成本过高** + - 候选扩展、排序与过滤成本放大 +- **shared/unshared KV 组织不正确** + - 导致显存浪费、重复读取或错误的 cache 选择 +- **host 参与过多** + - 使 multi-step 的收益被冲掉 +- **算子热路径收益不明显** + - 说明调度和执行形态还不够稳定 + +### 16.2 最值得观察的指标 + +即便不展开完整监控系统，这篇文档也至少应该把观察点写清楚： + +- 请求总时延 +- 不同 decode round 的耗时分布 +- scheduler 相关开销占比 +- host-device 往返次数或同步点 +- beam search / cache select 在总耗时中的占比 +- 显存占用和碎片化趋势 + +### 16.3 排障时最短路径 + +如果线上效果和预期不一致，建议优先按下面顺序排： + +1. 先确认请求是不是确实符合 fixed-step 场景假设； +2. 再确认 decode 主链是否真的走到了 multi-round pipeline； +3. 再看 beam search 和 cache select 是否成为主成本； +4. 最后再看 xAttention / KV 组织是否真的减少了重复加载和拷贝。 + +## 17. 基准与验收协议 + +为了让这篇文档后续更容易被用作评审和验收依据，这里补一个轻量的基准与验收协议模板。 + +### 17.1 建议至少覆盖的 workload + +- **长输入 / 短输出** + - 典型生成式推荐主场景 +- **不同 beam_width** + - 例如中等 beam 和大 beam 两组 +- **不同 top_k** + - 用于观察搜索和过滤成本的放大趋势 + +### 17.2 建议至少对比的指标 + +- 吞吐 +- P50 / P95 / P99 时延 +- 不同 decode round 的耗时 +- scheduler 开销占比 +- host 参与开销 +- 显存峰值 + +### 17.3 验收关注点 + +- fixed-step 是否真的降低了调度扰动 +- multi-step 是否真的降低了 host 回合控制开销 +- `xAttention` 是否真的改善了 shared prefix 的重复访问 +- beam 相关优化是否真的降低了搜索与过滤开销 + +这部分内容的价值在于：它能把“这套设计听起来合理”进一步收敛为“这套设计应该如何被验证”。 + +## 18. 关键参数与建议范围 + +为了避免这篇文档停留在抽象讨论层面，最后补一个轻量但实用的参数章节。这里不追求给出唯一正确的数值，而是整理出后续继续调优时最值得关注的参数范围和影响方向。 + +### 18.1 `beam_width` + +`beam_width` 决定每一轮保留的候选分支数，是生成式推荐 decode 成本最敏感的参数之一。 + +- `beam_width` 偏小： + - 搜索成本低 + - 候选覆盖较弱 + - 更容易得到较低时延 +- `beam_width` 偏大： + - 候选覆盖更强 + - 排序、过滤、cache select 成本会明显上升 + - fixed-step 和 multi-step 的收益会更突出，但同时更容易放大 decode 侧主成本 + +因此，`beam_width` 不是简单的“越大越好”或“越小越快”，而是推荐质量、时延和系统开销之间的核心平衡点。 + +### 18.2 `top_k` + +`top_k` 决定每个 beam 在一轮中扩展多少候选。 +在大候选池场景下，`top_k` 增长会直接放大： + +- 排序工作量 +- 无效 item 路径过滤工作量 +- 候选集合的中间态管理成本 + +如果 `top_k` 设计得过大，而 item 过滤又不够早，那么会导致很多计算都浪费在最终不会进入下一轮的候选上。 + +### 18.3 decode 轮数 + +decode 轮数越固定，fixed-step 调度和 multi-step 执行就越容易稳定发挥作用。 +如果 decode 轮数稳定在较小范围内，那么： + +- 设备侧更容易提前分配相关结构； +- `step_meta` 更容易保持稳定； +- 整图执行的收益更容易兑现。 + +反过来，如果 decode 轮数本身开始变得不稳定，那么 fixed-step 和 multi-step 的设计前提就会被削弱。 + +### 18.4 多流并发数量 + +多流不是越多越好。流数量过少，不能有效缓解等待；流数量过多，又可能带来新的调度竞争、资源抢占和额外管理开销。 + +因此，比较合理的调优方式通常是： + +1. 先确认 fixed-step 是否已经稳定； +2. 再评估 host 侧等待和排队问题是否明显； +3. 最后再增加多流或多执行 pipeline 数量。 + +### 18.5 参数调优的顺序建议 + +如果后续要继续做性能调优，建议按下面顺序看参数： + +1. 先看 decode 轮数是否稳定； +2. 再看 `beam_width` 是否合理； +3. 再看 `top_k` 是否过大； +4. 最后再看多流并发和系统层 overlap 是否足够。 + +这个顺序的原因是：前两个参数决定的是 workload 形状，后两个参数更多是在既定 workload 上做系统优化。 + +## 19. 参考资料 + +这篇文档尽量立足当前分支实现，但一些背景问题、调度思路和 memory / graph 执行语境，本身就来自公开资料和已有模型实践。后续如果要继续扩写，可以优先参考下面这些公开资料。 + +### 19.1 调度与服务系统 + +- Orca: A Distributed Serving System for Transformer-Based Generative Models + - 用于理解 continuous batching 的问题背景 + - 适合作为 fixed-step 对照材料 + +### 19.2 显存与 KV Cache + +- Efficient Memory Management for Large Language Model Serving with PagedAttention + - 用于理解 block-based KV 管理的常见问题 + - 适合作为 shared/unshared KV 设计的对照背景 + +### 19.3 生成式推荐模型 + +- OneRec + - 用于理解生成式召回模型的结构特点 +- OneTrans + - 用于理解生成式精排模型的结构特点 + +### 19.4 图执行与多步执行 + +- 仓库内已有 `Graph Mode` 设计文档 + - 适合理解 graph capture / replay、参数化和 Piecewise Graph +- 当前生成式推荐设计文档 + - 更适合理解 fixed-step、多轮 decode、beam search 和 xAttention 在推荐场景里的组合关系 + +### 19.5 使用建议 + +如果后续还要继续扩写这篇文档，我建议： + +- 需要写调度背景时，先看 Orca； +- 需要写 KV / memory 组织问题时，先看 PagedAttention； +- 需要写推荐模型结构时，先回到 OneRec / OneTrans； +- 需要写当前分支落点时，再回到本文档和代码锚点。 + +这样能保证文档继续变厚时，内容来源清楚，而且不会变成无边界的泛化解释。 + diff --git a/docs/src/content/docs/zh/design/graph_mode_design.md b/docs/src/content/docs/zh/design/graph_mode_design.md new file mode 100644 index 0000000000..c514580aaf --- /dev/null +++ b/docs/src/content/docs/zh/design/graph_mode_design.md @@ -0,0 +1,478 @@ +--- +title: "Graph Mode 设计文档" +sidebar: + order: 1 +--- +## 概述 + +xLLM 的 Graph Mode 覆盖多种图执行后端。其目标是在推理服务场景中，将原本由 Host 频繁发起的小粒度 kernel 启动，转化为先捕获、后重放的图执行流程，从而降低 Host 调度开销、减少设备气泡，并提升吞吐和时延稳定性。 + +本文档面向需要理解实现原理和关键设计的开发者，重点介绍以下内容： + +- Graph Mode 的原理与在 xLLM 中的落地方式 +- 动态维度参数化 +- Piecewise Graph +- 多 Shape 复用的显存池（含输入 Tensor 复用） + +本文聚焦 Graph Mode 的统一设计，不展开不同后端之间的差异。 + +本文档的设计目标包括： + +- 统一 xLLM 在不同图执行后端上的 Graph Mode 抽象 +- 解释动态维度参数化、Piecewise Graph 与多 Shape 显存复用三项关键设计 +- 说明这些设计分别解决什么问题、依赖什么前提、边界在哪里 + +本文档的非目标包括： + +- 不覆盖所有算子或所有模型的适配细节 +- 不替代功能文档中的参数说明与使用示例 + +相关设计文档： + +- 若希望看一个更偏业务推理场景、并且聚焦固定调度、多步执行和定制算子的案例，可参考：[生成式推荐设计文档](/zh/design/generative_recommendation_design/) + +## 1. Graph Mode 原理和在 xLLM 中的落地 + +### 1.1 Graph Capture / Replay 的基本原理 + +传统 eager 执行模式下，模型一次 forward 会由 Host 连续发起大量 kernel、memcpy 和同步操作。对于 decode 这类单 step 计算较小但请求频繁的场景，Host 调度开销会比较显著，设备端也更容易出现执行气泡。 + +Graph Mode 的基本思路是： + +1. **Capture 阶段**：第一次遇到某个 shape bucket 时，在专用 stream 上执行一次 forward，并把这条执行路径上的 kernel 启动、内存操作和依赖关系记录成图。 +2. **Replay 阶段**：后续相同 bucket 的请求，不再由 Host 逐个下发 kernel，而是直接重放已捕获的图。 + +![alt text](../../assets/graph_mode.png) + +这类机制通常要求： + +- **执行路径稳定** + - graph capture 记录的是一次具体执行路径；capture 完成后，这条路径上的控制流、launch 形态和依赖关系就被固化下来 + - 因此，capture 路径中无法在 replay 时再切换到另一套动态条件分支 +- **关键 Tensor 地址稳定** + - capture 期间写入图中的关键 Tensor 地址，在 replay 时必须仍然可用 +- **算子与结果语义稳定** + - replay 的正确性依赖路径上的算子与 Graph Mode 兼容，且不会因为运行时条件变化而改变语义 + +### 1.2 xLLM GraphMode 基础工作 + +从 1.1 的三条基本要求出发，前两项主要由 xLLM 运行时负责：一是把动态请求整理成可稳定 replay 的执行单元，二是保证 replay 时仍然访问 capture 阶段记录下来的固定地址。为此，xLLM 在 Graph Mode 下引入了统一的 Graph Executor，负责收敛分桶、持久化 buffer、graph cache 以及 capture / replay 生命周期管理。 + +xLLM 在运行时侧需要完成的基础工作，主要包括： + +- **围绕执行路径稳定的图选择与执行调度** + - 请求需要先按 `num_tokens` 或相近 shape 归入 bucket + - bucket 维度维护 graph cache：已命中则直接 replay，未命中则先 capture 再缓存 + - 对可整图捕获的路径执行完整图；存在 break graph 的路径则切换到 Piecewise Graph +- **围绕地址稳定的持久化 buffer 与 graph instance 管理** + - tokens、positions、seq_lens、block_tables 等动态输入不能在 replay 前重新分配到任意新地址， xLLM 需要先把这些输入写入持久化 buffer，再在固定地址上更新内容 + - 模型计算图过程临时分配的Tensor都需要持久化保存对对应的graph instance中，不能随着作用域结束触发Tensor回收 + - 启用共享内存池后，不同 shape 的 capture 可以复用同一组底层物理内存，但 graph 看到的虚拟地址仍需保持稳定 +- **统一的 Graph Executor 封装** + - capture、cache、replay、图实例管理以及后端差异屏蔽，都需要由运行时统一处理 + - 这层封装的目的，是把 Graph Mode 的运行时组织逻辑从模型实现中剥离出来，而不是把它们散落到各个 layer 或算子里 + +在具体执行上，Graph Executor 的运行时流程可以概括为： + +1. **请求分桶**：按 `num_tokens` 或相近 shape 归类到 bucket +2. **输入准备**：将 tokens、positions、seq_lens、block_tables 等写入持久化 buffer，并更新 `attn_metadata`、`plan_info` 等动态元数据 +3. **capture 或 replay 决策**：bucket 已命中则直接 replay；未命中则先 capture、缓存图，再进入 replay 路径 +4. **执行图**：按场景执行完整图或 Piecewise Graph + +而 1.1 中第三项“算子与结果语义稳定”，更多落在模型和算子本身的适配上，见 1.3。 + +### 1.3 模型适配 Graph Mode 需要的改造 + +在 xLLM 运行时负责分桶、buffer 和 graph instance 管理之后，模型侧仍然需要解决 1.1 中第三项要求：保证 capture / replay 期间的算子行为与结果语义稳定，不会因为 host 参与、dispatch 变化或 launch 形态变化而失效。 + +模型适配 Graph Mode 需要的改造主要包括： + +- **去掉不支持 capture 的操作** + - 泛化要求：capture 路径中不能包含会触发 host 参与决策、host-device 隐式同步或额外控制流切换的操作，例如 stream 同步、读取 host tensor 决定逻辑分支、隐式 host-device 同步交互等 + - 典型示例 1：算子内部依赖 host tensor 内容决定任务划分或执行模式。例如商发 8.3 的 ATB [PA](https://www.hiascend.com/document/detail/zh/canncommercial/83RC1/API/ascendtbapi/ascendtb_01_0197.html) 和 [MLA](https://www.hiascend.com/document/detail/zh/canncommercial/83RC1/API/ascendtbapi/ascendtb_01_0314.html)，会根据 host 侧kv_seq_lens和q_seq_lens来做不同的任务排列。 + - 典型示例 2：host 标量参与 Tensor 计算，或 host 侧直接读取 Tensor 数据，例如： + +```cpp +Tensor a; +Tensor b = a * 0.5; // 标量会被隐式传到 tensor 所在设备，产生同步 h2d + +torch::Tensor max_of_seq = torch::max(input_params.kv_seq_lens); +max_seq_len_ = std::max(max_of_seq.item(), max_seq_len_); // host 读取 tensor +``` + +- **算子适配** + - 泛化要求：需要保证计算图的 kernel 执行路径稳定。对于同一个输入 shape，每次执行计算图选择的 kernel 应保持一致，不发生 prefill、chunked prefill、decode 混杂；同时 `grid_dim`、`block_dim`、task 数量、workspace 形态和 tiling 内容也不能变化 + - 典型示例：Ascned Cann商发8.3版本 的 ATB PA 会根据序列长度和 batch size 实时决定是否启用 flash-decoding 长序列模式；长序列模式和短序列模式的 tiling key 不同，dispatch 后实际 launch 的 `kernel_name` 也不同。这种“同一业务路径、不同运行时条件触发不同 kernel”的行为，在进入 Graph Mode 前需要先收敛 + +## 2. 动态维度参数化 + +### 2.1 问题 + +Graph Capture 记录的是固定的 kernel 执行序列和 launch 形态。Replay 时，运行时只能重复这套已经记录下来的 `grid_dim`、`block_dim`、task 数量和 tiling 路径，不能在同一张图里重新决定它们。 + +因此，请求里的动态维度进入 Graph Mode 后，并不都能在 replay 时继续变化。对 attention 来说，真实动态因素通常不止 `num_tokens`，还包括 `batch_size`、`q_seq_lens`、`kv_seq_lens`、`block_tables_size` 等。如果这些维度进一步影响 task 划分、workspace 布局、`tiling_params` 或 kernel 路径，那么 replay 就会沿用 capture 时的旧配置，从而出错。 + +本章要解决的问题就是：哪些动态信息可以在图选定后继续更新，哪些必须通过分桶或重新建图处理。 + +### 2.2 解法 + +动态维度参数化的解法，不是“让一张图接受任意 `num_tokens`”，而是在**图已经按 `bucket_num_tokens` 选定之后**，让同一个 bucket 尽量覆盖更多真实动态请求。 + +#### 2.2.1 参数化边界 + +一个维度能不能在 replay 时变化，关键不在于它是不是“请求里的动态值”，而在于它有没有在 capture 阶段被折叠进图的执行形态。 + +可以把动态因素分成两类： + +1. **决定图形态的维度** + - 一旦进入 `grid_dim`、`block_dim`、task 数量、workspace 布局、tiling key 或 `tiling_params`，它就成为 capture 结果的一部分 + - 这类维度变化后，通常不能在同一张 graph 内直接修改 + +2. **图选定后仍可更新的维度** + - 例如 `batch_size`、`q_seq_lens`、`kv_seq_lens`、`block_tables`、`new_cache_slots`、`plan_info` + - 它们只有在不改写执行形态时，才适合作为参数化对象 + +一个简化后的 norm kernel 例子是： + +```cpp +int grid_dim = num_tokens; +NormKernel<<>>(x, y, ...); +``` + +这里的 `grid_dim`、`block_dim` 都属于 launch params。只要 `num_tokens` 已经参与了 launch 配置，它就属于图形态的一部分；capture 完成后，replay 就不能在同一张图里把 `grid_dim = 128` 改成 `grid_dim = 256`。 + +因此，动态维度参数化的判断标准很直接：凡是会改变 launch、workspace 或 tiling 路径的动态因素，都不应只靠 replay 前改参数解决。 + +#### 2.2.2 xLLM 中的处理方式 + +xLLM 的处理方式分两层。 + +第一层先处理 `num_tokens`：通过 bucket 化和 padding，把原始请求归一化到固定的 `bucket_num_tokens`，再据此选定图。 + +第二层再处理 bucket 内剩余的动态信息：对依赖 host tensor、host planning 或 host 侧 tiling 的算子做改造，把真正随请求变化的信息放到设备侧持久化 buffer 中，或者显式外置为 replay 前可更新的数据，而不是让 replay 继续绑定 capture 时那次 host 侧 planning 结果。 + +以 NPU attention 为例，改造前常见的问题是：`q_seq_lens`、`kv_seq_lens` 先在 Host 侧参与 planning，生成 task、workspace 或 tiling 相关数据，再发射 kernel： + +```cpp +// Host side +auto plan_info = PlanAttention(q_seq_lens_host, kv_seq_lens_host, ...); +WritePlanToWorkspace(attention_workspace, plan_info); + +AttentionKernel<<>>( + q, k, v, out, + attention_workspace, + tiling_params); +``` + +这类实现里，graph replay 不会自动重做 planning，因此只能重复 capture 时那次 launch 和那次 workspace 布局。改造后的方向是，在固定 launch 形态下，把 `q_seq_lens`、`kv_seq_lens`、`block_tables_size`、`plan_info` 等动态信息写入设备侧持久化 buffer，再由 kernel 在执行时读取，例如`kv_seq_lens`的参数化示例： + +```cpp +struct AttentionLaunchArgs { + void* q; + void* k; + void* v; + void* out; + int32_t batch_size; + int32_t* kv_seq_lens; // device buffer + void* attention_workspace; + void* tiling_params; +}; + +LaunchAttentionKernel(args); +``` + +如果某个动态因素一旦变化就会改写任务数量、数组长度、workspace 形态或 tiling 路径，那么 xLLM 不会强行把它塞进“参数化更新”里，而是采用两种处理方式之一： + +1. 扩大分桶维度，把这些因素纳入选图键，例如 `num_tokens` 和 `num_comm_tokens` 的组合 +2. 把这段对动态 shape 敏感的执行留在 full graph 之外，交给 Piecewise Graph 处理 + +### 2.3 结果 + +动态维度参数化的直接结果是：同一个 `num_tokens` bucket 可以安全覆盖更多真实请求；在执行形态稳定的前提下，attention 等算子也能在 replay 时看到正确的 `batch_size`、`seq_lens`、`block_tables_size`、`plan_info` 等动态信息。 + +需要注意的是，参数化并不等于取消分桶。`num_tokens` 的动态性仍主要通过选图吸收；如果同一个 bucket 下还有其他会改写 tiling key、任务数量、通信规模或 kernel 路径的因素，那么这些因素仍然必须进入分桶键，或者转为 Piecewise Graph。通信相关场景就是典型例子，例如 Attention DP + MoE EP 下，通信规模可能取决于最大 DP Data Size，而不完全等于单卡本地的 `num_tokens`。 + +## 3. Piecewise Graph + +### 3.1 问题 + +在 prefill、chunked prefill 等场景中，整条 forward 路径往往比 decode 更复杂。某些算子可能： + +- 无法被稳定 capture +- 对动态元数据高度敏感 +- 在 capture 模式下依赖额外的运行时准备逻辑 + +如果要求“整图都必须可捕获”，Graph Mode 的适用范围会非常受限。只要有一个关键算子 break graph，整条路径都无法使用图执行。 + +### 3.2 解法 + +#### 3.2.1 底层逻辑 + +在实现上，一个经常被问到的问题是：为什么 decode 可以使用 full graph，而 chunked prefill 往往只能使用 Piecewise Graph？ + +关键差别在 attention。 + +在当前 decode 语义下，单步通常是“每个序列生成一个 token”，因此 `num_tokens` 与 `batch_size` 基本是一一对应的。对 attention 来说，这意味着： + +- `num_tokens` 增长时，`batch_size` 也同步增长 +- `q_seq_lens` 的模式相对稳定 +- 任务划分、workspace 需求和 launch args 变化主要可以跟着 `num_tokens` 一起分桶 + +因此，按 `num_tokens` 建 bucket 时，decode 路径满足 full graph 的复用条件。 + +但在 chunked prefill 中，`num_tokens` 和 `batch_size` 不再绑定。相同的 `num_tokens`，可能对应完全不同的请求组合，例如： + +- `num_tokens = 128, batch_size = 2, q_seq_lens = [64, 64]` +- `num_tokens = 128, batch_size = 4, q_seq_lens = [32, 32, 32, 32]` +- `num_tokens = 128, batch_size = 8, q_seq_lens = [16, 16, 16, 16, 16, 16, 16, 16]` + +对 attention 算子来说，这些请求虽然 `num_tokens` 相同，但并不是同一种 shape。attention 的任务数量划分、本地索引、`q_seq_lens` / `kv_seq_lens` 的长度、以及对应的 `plan_info` 都和 `batch_size` 有直接关系。如果系统只按 `num_tokens` 去复用之前记录的 graph，那么 attention 在 replay 时就可能沿用 capture 阶段那套“旧的 batch 划分”和“旧的任务分块”，从而导致结果错误。 + +这里的 `batch_size` 之所以难以简单参数化，是因为它影响的不只是某个标量值，而是整组 launch args 的结构： + +- `q_seq_lens` / `kv_seq_lens` 数组长度会变 +- attention 的任务数量和任务边界会变 +- `plan_info` 和 workspace 布局会变 +- kernel 内部使用的索引和分块逻辑会变 + +换句话说，`batch_size` 在 chunked prefill 中不是“改几个参数就行”的维度，而是会改变 attention 这一段图的执行形态。除非把 `num_tokens × batch_size` 甚至更细粒度的 `seq_lens` 组合都纳入分桶，也就是做高维笛卡尔积建图，否则仅靠 `num_tokens` 选图并不安全。 + +这也是为什么当前实现更适合： + +- **decode**：attention 可以随 full graph 一起捕获 +- **chunked prefill**：attention 留在图外，其他稳定部分进入 Piecewise Graph + +#### 3.2.2 核心思路 + +Piecewise Graph 的目标不是强行整图捕获，而是把一条完整执行路径拆成多个 piece： + +- 可图化的部分：捕获为多个子图 +- 不适合图化的部分：保留为 eager 形式 + +在 replay 时，再按 capture 期间记录的顺序，把这些 piece 串起来执行。 + +这样可以在“无法整图”的前提下，尽量保留 Graph Mode 的收益。 + +下图以三层 Qwen3 decoder 为例展开了 Piecewise Graph 的执行顺序：attention 保留为独立 runner，而两次 attention 之间的非 attention 算子会被尽量压紧并组织成连续的 graph piece。 + +![Piecewise Graph 示意图](../../assets/piecewise_graph_diagram.svg) + +#### 3.2.3 xLLM 中的实现方式 + +在 xLLM 中，Piecewise Graph 主要用于 prefill 类场景，由 `enable_prefill_piecewise_graph` 控制。 + +它的核心实现方式是： + +1. 使用 `PiecewiseGraphs` 维护 replay 指令序列 +2. 可捕获片段以图对象形式保存 +3. attention 这类不直接进入图的部分，通过 `AttentionRunner` 记录为 runner +4. replay 时按原始顺序执行 `graph -> runner -> graph -> ...` + +这套机制的关键点在于 attention 的处理： + +- 在 piecewise capture 期间，attention 会暂时结束当前图捕获 +- attention 本身不在 capture 时执行，而是把运行 attention 所需的张量、workspace 和参数保存下来 +- replay 时，再基于最新的 `plan_info`、`q_cu_seq_lens`、`kv_cu_seq_lens` 执行 attention + +这样做的直接原因就是：chunked prefill 下 attention 的动态维度不只由 `num_tokens` 决定，而当前图分桶主要按 `num_tokens` 组织。把 attention 留在图外，才能避免“同样的 `num_tokens` 命中了 graph，但 `batch_size` 和 `seq_lens` 已经变了”的 replay 错误。 + +因此，Piecewise Graph 的本质不是“图内支持 attention 的所有动态逻辑”，而是： + +- 让可稳定 capture 的部分进入图 +- 让对动态 shape 更敏感的 attention 保持单独执行 +- 通过统一指令序列保证整条执行路径顺序不变 + +### 3.3 结果 + +这套做法带来的结果和边界主要体现在以下几个方面。 + +#### 3.3.1 适用场景 + +- prefill +- chunked prefill +- 局部 break graph，但整条路径仍希望获得图执行收益的场景 + +#### 3.3.2 优点 + +- 比“整图要么全成功、要么全放弃”更灵活 +- 能显著扩大 Graph Mode 的覆盖范围 +- 适合 attention 动态性强、而 MLP 等部分较稳定的模型结构 + +#### 3.3.3 限制 + +- 当前主要覆盖 prefill 类路径 +- replay 依赖正确更新的 `attn_metadata.plan_info` + +## 4. 多 Shape 复用的显存池 + +### 4.1 问题 + +Graph Mode 在多 shape 场景下面临两个同时出现的问题： + +1. **捕获期显存随 shape 数量线性增长** + - 每次新的 shape 进入 capture，运行时通常会为它分配独立的 memory buffer pool + - 由于图执行要求 capture 和 replay 之间地址稳定，这些缓冲区往往不能及时释放 + - 结果是显存占用从期望的 `max(shape)` 变成了 `sum(shape)` + +2. **输入侧也会重复占用显存** + - 如果每个 shape 都维护一套独立的 tokens、positions、seq_lens、block_tables 等输入 tensor + - 那么即使图本身可以复用，输入侧显存也会继续按 shape 累积 + +因此，多 Shape 复用显存池要解决的问题，不只是节省显存，而是要在支持动态新 shape 的同时，把显存行为从 `sum(shape)` 收敛到 `max(shape)`。 + +具体目标包括： + +1. **捕获期显存复用**：不同 shape 的 graph 共享同一组底层物理内存 +2. **输入侧显存复用**：不同 shape 的 replay 共享同一组持久化输入 buffer +3. **地址不冲突**：每个已捕获 graph 仍然保有自己稳定的虚拟地址视图 +4. **支持按需新建 shape**：新 shape 出现时可以继续 capture，而不必推倒已缓存的 graph +5. **不增加 replay 开销**：内存复用主要发生在 capture 和输入准备阶段，replay 路径尽量保持不变 + +#### 4.1.1 根本原因 + +- 以 CUDA 路径为例，allocator 往往会为每次 graph capture 分配独立的 memory buffer pool +- Graph Mode 又要求 capture 和 replay 之间地址稳定，因此旧 shape 对应的 buffer 不能像普通 eager 临时内存那样及时回收 +- 当不同 shape 持续进入 capture 时，显存占用就会从期望的 `max(shape)` 累积到 `sum(shape)` + +#### 4.1.2 约束条件 + +- 不同 shape 的 graph 在当前前提下不会同时 replay +- 需要支持动态出现的新 shape，按需进行 capture +- 不希望为了新 shape 推倒已缓存 graph，或重新捕获已经可用的旧 shape +- replay 路径不希望因为显存复用而引入额外的图执行复杂度 + +#### 4.1.3 失败方案与技术难点 + +一个更直接的想法是：每次 capture 前重置分配指针，让不同 shape 复用同一段 virtual address space。 + +但这条路在工程上并不安全。allocator 通常按地址跟踪已经分配的 block；如果新 capture 复用了旧地址，就可能覆盖原来的地址记录。这样一来，旧 graph 后续在 tensor 析构或释放时，就可能遇到地址记录缺失，甚至触发 `invalid device pointer` 一类错误。 + +这也暴露了问题的关键：Graph Mode 真正需要稳定的是“每张 graph 看到的地址视图”，而真正需要复用的是“底层 physical memory”。因此，技术难点不只是节省显存，而是要同时满足： + +- 已捕获 graph 的 virtual address space 彼此不冲突 +- 底层 physical memory 又能在多 shape 之间共享 +- 新 shape 进入 capture 时，不破坏已有 graph 的可 replay 性 + +### 4.2 解法 + +多 Shape 复用的底层思路，不是让不同 shape 共用同一段虚拟地址，而是让它们共享同一组物理内存，同时为每个已捕获 graph 保留独立的虚拟地址视图。 + +这样设计的原因是：如果强行让不同 shape 复用同一段虚拟地址空间，allocator 的地址跟踪会发生冲突，旧 graph 释放或析构时可能找不到原来的地址记录，最终导致错误。换句话说，真正需要复用的是底层物理显存，而不是 graph 看到的虚拟地址。 + +为了说明为什么 xLLM 选择“共享 physical memory + 独立 virtual address space”，可以把几种方案对比如下： + +| 方案 | 内存复用机制 | 物理内存结果 | 地址冲突风险 / 规避方式 | 主要局限 | +|------|--------------|--------------|--------------------------|----------| +| vLLM | 共享 graph memory pool + 大 shape 优先 capture | 可能继续接近 `sum(shape)` | 依赖共享 memory pool 和空闲块管理，通常无显式地址冲突 | 复用效果受捕获顺序和 allocator 行为影响 | +| SGLang | 全局 graph memory pool + 大 shape 优先 capture | 可能继续接近 `sum(shape)` | 依赖全局 pool / graph-private pools 管理规避冲突 | 不同 shape 较多时显存仍会继续增长 | +| xLLM | 多 virtual address space 映射同一组 physical memory | 收敛到 `max(shape)` | 每张 graph 保留独立地址视图，从根源避免地址记录冲突 | 依赖底层运行时提供 VMM 和 allocator 接入能力 | + +xLLM 的选择不是复用同一段 virtual address space，而是让每个 graph 保留自己的地址视图，同时把真正可共享的部分下沉到底层 physical memory。 + +#### 4.2.1 输入 Tensor 多 Shape 复用 + +xLLM 先解决输入侧显存重复占用的问题。做法是：不再为每个 shape 分配独立输入 tensor，而是预分配一组“最大 shape”对应的持久化 buffer，然后让不同 shape 共享这组 buffer。 + +这组 buffer 通常包括： + +- `tokens` +- `positions` +- `q_seq_lens` +- `kv_seq_lens` +- `block_tables` +- `new_cache_slots` +- `hidden_states` + +replay 前的处理方式也比较直接： + +1. 把当前请求的实际输入写入 buffer 前缀 +2. 必要时对 padding 区域补零 +3. 用 slice 视图构造当前 actual shape 的输入 + +这样做的结果是：输入侧显存不再按 shape 累加，而是稳定在一份 `max(shape)`。 + +#### 4.2.2 捕获期显存复用 + +解决输入侧之后，还需要解决 graph capture 本身的显存累积问题。 + +xLLM 的核心做法是： + +- 每次 capture 使用**新的虚拟地址空间** +- 不同虚拟地址空间映射到**同一组物理内存** + +这样可以同时满足两件事： + +1. **对 graph 来说地址稳定**：每个已捕获 graph 都看到自己的固定虚拟地址 +2. **对物理显存来说只保留一份**：底层物理内存按最大需求扩张，不再按 shape 叠加 + +相比“复用同一虚拟地址”的思路，这种做法规避了地址跟踪冲突问题，因此更适合和现有 allocator 机制协作。 + +这套方案依赖统一的虚拟内存管理能力，核心操作包括： + +- 保留一段新的虚拟地址空间 +- 创建或扩张底层物理内存 +- 将同一组物理内存映射到多个虚拟地址空间 +- 在需要时执行 unmap、释放地址空间和访问权限设置 + +其中最关键的特性是：**同一组物理内存可以同时映射到多个虚拟地址空间**。这正是“不同 shape 的 graph 地址互不冲突，但底层物理显存仍然共享”的基础。 + +#### 4.2.2.1 地址映射关系 + +Graph 捕获期显存复用的核心映射关系如下： + +![多虚拟地址空间映射同一物理内存](../../assets/shared_vmm_mapping_diagram.svg) + +这张图表达的是：随着 shape 逐步增大，每次 capture 都切换到新的虚拟地址空间，但三个 virtual space 复用的是同一个 physical memory pool。不同 shape 只会使用其中不同比例的页，而底层物理显存总占用收敛到 `max(shape)`。 + +### 4.3 结果 + +#### 4.3.1 内存效率指标 + +| 指标 | 显存不复用方案 | 显存复用方案 | +|------|----------------|--------------| +| 物理内存大小 | `sum(shape)` | `max(shape)` ✅ | +| 虚拟内存 | 多份 | 多份 | +| 稳定性 | 地址冲突风险 | 稳定 ✅ | + +这里的关键变化是：物理显存收敛到 `max(shape)`，而 virtual address space 仍会随着已捕获 shape 数量增长。 + +#### 4.3.2 性能影响 + +- **Capture 阶段**：切换虚拟地址空间和扩张映射会带来额外开销 +- **Replay 阶段**：无额外图执行开销，与常规 Graph Mode replay 基本一致 +- **输入准备阶段**：增加一次写入持久化 buffer 和 slice 视图构造，但这部分通常远小于重复分配带来的成本 + +#### 4.3.3 虚拟地址空间开销 + +这套方案的主要代价之一，是 virtual address space 会随着已捕获 shape 数量持续增长。但这部分增长对应的是地址空间占用，而不是物理显存占用。 + +在 64 位系统下，virtual address space 通常远大于单卡显存规模，因此对当前这类 graph capture 场景来说，它通常不是首要瓶颈。换句话说，这套方案是用更充足的地址空间，换取物理显存从 `sum(shape)` 收敛到 `max(shape)`。 + +#### 4.3.4 结果验证 + +- 不同 shape 的 capture 可以共享同一组底层 physical memory +- 每次 capture 仍保有独立的 virtual address space，而不是复用同一组地址视图 +- 地址冲突类问题不再通过“复用同一虚拟地址”这条路径暴露出来 +- 显存占用从按 shape 累积的 `sum(shape)` 收敛到 `max(shape)` + +#### 4.3.5 实际效果 + +- 多 shape 场景下，显存行为从“随 capture 次数线性累积”转为“物理显存按最大 shape 收敛” +- Graph 复用的稳定性提升，规避了直接复用同一虚拟地址空间带来的地址冲突问题 +- 设计上支持动态新 shape 按需 capture，而不必推倒已有 graph 缓存 +- replay 路径保持不变或基本不变，主要复杂度集中在 capture 和输入准备阶段 + +#### 4.3.6 适用前提与边界 + +- 虚拟地址空间会随着已捕获 shape 数量增长而增长，但物理显存只按最大需求扩张 +- 方案依赖底层运行时提供虚拟地址空间管理、物理内存映射以及 allocator 接入能力 +- 当前假设不同 shape 的 graph 不会同时 replay；正是这个前提让底层物理内存能够安全共享 +- 当前实现仍以生命周期内统一持有物理内存和虚拟空间为主，不做细粒度动态释放 + +## 5. 参考文档 + +- [ACLGraph Capture / Replay 机制说明](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/83RC1alpha002/appdevg/acldevg/aclcppdevg_000519.html) +- [CudaGraph Capture 机制参考](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#cuda-graphs) +- [vLLM 图执行实现参考](https://github.com/vllm-project/vllm/blob/main/vllm/v1/worker/gpu/cudagraph_utils.py) +- [SGLang 图执行实现参考](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/model_executor/cuda_graph_runner.py) + diff --git a/docs/src/content/docs/zh/dev_guide/ai_coding_workflow.md b/docs/src/content/docs/zh/dev_guide/ai_coding_workflow.md new file mode 100644 index 0000000000..629f6474f8 --- /dev/null +++ b/docs/src/content/docs/zh/dev_guide/ai_coding_workflow.md @@ -0,0 +1,103 @@ +--- +title: "AI Coding 工作流" +sidebar: + order: 4 +--- + +本文总结一套面向 xLLM 开发者的实践工作流，用于 NPU 推理优化、回归定位，以及为 +PR 准备可复现的验证证据。 + +完整 workflow 知识库维护在 +[:simple-github: xllm-workflow](https://github.com/xLLM-AI/xllm-workflow)。 +当你需要可直接加载的 agent skills、Prompt 模板、artifact schemas 或模型优化历史时， +可以使用该仓库。 + +![xLLM AI Coding 工作流](../../assets/xllm-ai-coding-workflow-zh.png) + +## 什么时候使用 + +当改动需要普通代码 review 之外的证据时，建议使用这套工作流： + +- 优化 TTFT、TPOT、TPS、内存使用或 serving 并发能力； +- 公平比较 xLLM 与 vLLM-Ascend、SGLang NPU 或其他 serving 框架； +- 定位乱码输出、数据集掉分、GPU/NPU 不一致、OOM、图 replay 失败、HCCL 问题或运行时 crash； +- 在 NPU 相关 PR 合入前做验证； +- 判断是否需要算子迁移或 kernel-level 实验。 + +不要把一次 smoke run 当作正式结论。正式的性能和精度结论应包含精确命令、环境、 +workload、原始 artifacts 和归一化 summary。 + +## 证据闭环 + +对性能优化和 correctness-sensitive 改动，推荐遵循下面的闭环： + +```text +target -> baseline -> profiling -> patch -> accuracy -> performance -> record +``` + +核心原则是把 benchmark、profiling 和 accuracy 证据分开。Profiling 用于解释瓶颈， +不能替代 warmed-up before/after 性能对比。 + +完整优化任务可以按下面阶段推进： + +| 阶段 | 目的 | 产物 | +|---|---|---| +| 目标与环境 | 定义目标、模型、框架 commit、硬件、CANN/runtime 版本、workload 和 SLA。 | Run manifest | +| 历史知识 | 查询历史模型 PR、失败尝试和已知风险路径。 | History notes | +| 公平基线 | 改代码或参数前先跑 warmed-up baseline。 | 原始 metrics 和 summary | +| 证据采集 | 根据症状采集 profiling、capacity、pipeline、compute 或 accuracy 证据。 | 诊断报告 | +| Patch | 每轮尽量只做一个有意义、可 review 的改动。 | Code diff | +| 验证 | 根据改动重新运行 accuracy、performance、build 和 UT 检查。 | 验证表 | +| 沉淀 | 保存命令、指标、失败尝试、风险说明和后续工作。 | 可复用经验 | + +## 开发者检查清单 + +在提交或更新 NPU 优化 PR 前，PR 描述应能回答这些问题： + +- 使用了什么模型、tokenizer、dtype、设备型号、设备数量和框架 commit？ +- 精确的启动命令和 benchmark 命令是什么？ +- baseline 是否经过 warmup，并且运行在干净设备上？ +- profiling 结果是否只作为诊断证据使用？ +- 跑到了哪个精度验证等级？如果有失败样例，保存在哪里？ +- patch 改了什么？还剩哪些风险？ +- 其他开发者可以用哪些 artifacts 复现结论？ + +## 推荐产物 + +正式结果建议在同一个 run root 下保存： + +- 包含环境和命令细节的 `manifest.md` 或 `manifest.yaml`； +- 原始 evalscope 或 benchmark 输出； +- 归一化的 `metrics.json` 或 `summary.md`； +- 使用 profiling 诊断时的 profiling report 和 timeline notes； +- 精度任务中的 `failed_cases.jsonl` 或等价坏例记录； +- 说明改了什么、为什么安全、如何验证的 PR notes。 + +workflow 仓库提供了这些共享 schema： + +- `references/run-manifest-template.md` +- `references/perf-artifact-schema.md` +- `references/profiling-artifact-schema.md` +- `references/accuracy-artifact-schema.md` + +## 相关 Skills + +workflow 仓库包含面向任务的 skills，可由 Codex、Claude Code、opencode 或其他本地 +agent runtime 加载： + +| 任务 | Skill | +|---|---| +| 端到端优化 | `xllm-npu-sota-loop` | +| 服务启动和 evalscope 采集 | `xllm-npu-eval-runner` | +| 公平框架对比 | `xllm-npu-benchmark` | +| msprof / MindStudio 分析 | `xllm-npu-profiler` | +| Decode bubble 和 rank skew 分析 | `xllm-npu-pipeline-analysis` | +| HBM、KV cache 和 OOM 分析 | `xllm-npu-capacity-planner` | +| FLOPs、MFU 和理论下界估算 | `xllm-npu-compute-simulation` | +| 精度回归定位 | `xllm-npu-accuracy-debug` | +| Crash 或运行时事故定位 | `xllm-npu-incident-triage` | +| NPU 代码 review | `xllm-npu-code-review` | +| 算子迁移 | `xllm-npu-op-migration` | + +这些 skills 是工程纪律的辅助工具。最终结论仍应基于可复现的 xLLM artifacts 和可 +review 的代码改动。 diff --git a/docs/src/content/docs/zh/dev_guide/code_arch.md b/docs/src/content/docs/zh/dev_guide/code_arch.md new file mode 100644 index 0000000000..0d8ce5b124 --- /dev/null +++ b/docs/src/content/docs/zh/dev_guide/code_arch.md @@ -0,0 +1,30 @@ +--- +title: "代码结构" +sidebar: + order: 1 +--- +``` +├── xllm/ +| : 主代码目录 +│ ├── api_service/ # api服务化实现 +│ ├── core/ +│ │ : xllm核心功能代码目录 +│ │ ├── common/ +│ │ ├── distributed_runtime/ # 分布式PD服务实现 +│ │ ├── framework/ # 引擎执行模块实现 +│ │ ├── kernels/ # 国产芯片kernels适配实现 +│ │ ├── layers/ # 模型层实现 +│ │ ├── platform/ # 多平台兼容层 +│ │ ├── runtime/ # worker/executor角色实现 +│ │ ├── scheduler/ # 批调度与PD调度实现 +│ │ └── util/ +│ ├── function_call # function call实现 +│ ├── models/ # 模型实现 +│ ├── processors/ # 多模态模型预处理实现 +│ ├── proto/ # 通信协议 +│ ├── pybind/ # python接口 +| └── server/ # xLLM服务实例 +├── examples/ # 服务调用示例 +├── tools/ # NPU Timeline生成工具 +└── xllm.cpp # xLLM启动入口 +``` diff --git a/docs/src/content/docs/zh/dev_guide/online_profiling.md b/docs/src/content/docs/zh/dev_guide/online_profiling.md new file mode 100644 index 0000000000..715ecb9693 --- /dev/null +++ b/docs/src/content/docs/zh/dev_guide/online_profiling.md @@ -0,0 +1,105 @@ +--- +title: "在线性能采集 (Online Profiling)" +sidebar: + order: 3 +--- + +## 背景 + +在线服务部署中，Timeline 性能采集对于定位性能瓶颈至关重要：它能展示时间究竟消耗在主机侧调度、设备侧算子还是通信上。vLLM 通过两个 HTTP 接口 `POST /start_profile` 与 `POST /stop_profile` 实现该能力，在每个 worker 上开启/关闭采集，从而无需重启即可在运行中的服务上采集 timeline。 + +xLLM 提供等价的能力，并通过 `--profile_backend` 选择两种后端： + +- **`torch`（默认）** —— 通过 libtorch 的 Kineto profiler（即 `torch.profiler.profile` 的 C++ 等价实现）在进程内采集 CPU 与 CUDA 活动，并在 `/stop_profile` 时把 Chrome trace 直接写到磁盘。无需任何外部 profiler：正常启动服务、调用这两个接口即可。该方式与 vLLM 默认的 `TorchProfilerWrapper` 对齐。 +- **`cuda`** —— 仅开关 CUDA profiler 的 capture range（`cudaProfilerStart()` / `cudaProfilerStop()`）。它本身不记录任何内容，必须与 NVIDIA Nsight Systems（`nsys`）配合使用：服务在 `nsys profile` 下启动，并将 capture range 绑定到 CUDA Profiler API，这两个接口负责开启/关闭 `nsys` 实际记录的窗口。 + +两种后端目前均仅支持 CUDA，其他芯片后续支持。 + +## 原理介绍 + +整体调用链复用了已有的 `sleep`/`wakeup` 广播路径： + +``` +HTTP POST /start_profile + -> APIService::StartProfileHttp (xllm/api_service) + -> Master::start_profile (xllm/core/distributed_runtime) + -> Engine::start_profile (广播到所有 worker) + -> WorkerClient::start_profile_async (进程内本地 worker) + -> RemoteWorker::start_profile_async (通过 brpc 的远程 worker) + -> WorkerService::StartProfile (worker 服务端处理) + -> WorkerImpl::start_profile + -> TorchProfiler::start (Kineto，默认) + 或 CudaProfiler::start (cudaProfilerStart，--profile_backend=cuda) +``` + +Engine 会将请求并发广播到每个 worker，并等待全部确认。由于 xLLM 在单进程内为每个设备运行一个 worker 线程，而 Kineto/CUPTI 是进程级的，因此每个 profiler 都被封装为进程级、幂等的单例：首次 `start` 会为整个进程开启采集窗口，重复或并发的 `start`/`stop` 调用会被合并处理。`/stop_profile` 会关闭采集窗口。 + +对于 `torch` 后端，profiler 在 worker 的计算线程（即执行 forward 的线程）上开启与关闭，从而能采集到主机侧的 CPU 算子。在 `/stop_profile` 时，由 libtorch 自行写出 Chrome trace，文件落到 `--profile_dir`（未设置时为当前工作目录）。对于 `cuda` 后端，trace 的输出位置由 `nsys` 控制（其 `-o` 参数），而非由 xLLM 管理。 + +## 使用方式 + +该功能默认关闭，需显式开启。启动服务时开启采集： + +```shell +--enable_online_profile=true +``` + +`enable_online_profile`、`profile_backend` 与 `profile_dir` 也可以通过 JSON 配置文件设置。仅当 `--enable_online_profile=true` 时这两个接口才会生效；否则会返回错误并提示如何开启该功能。 + +### 默认后端（`torch`，无需 nsys） + +正常启动服务，并指定 trace 的输出目录即可： + +```shell +<你的 xllm serve 启动命令> \ + --enable_online_profile=true \ + --profile_dir=/path/to/traces # 可选；默认为当前目录 +``` + +随后，针对运行中的服务： + +```shell +# 开始采集 +curl -X POST http://127.0.0.1:9977/start_profile + +# ... 发送需要采集的推理请求 ... + +# 停止采集（此时写出 trace） +curl -X POST http://127.0.0.1:9977/stop_profile +``` + +（请将 host/port 替换为你的服务地址。）在 `/stop_profile` 时，每个 worker 会在 `--profile_dir` 下写出名为 `xllm_rank__.pt.trace.json` 的 Chrome trace，其绝对路径会打印在服务日志中。 + +### `cuda` 后端（仅 capture range，需要 nsys） + +若改用 capture range 方式，设置 `--profile_backend=cuda`，并在 `nsys` 下启动服务、将 capture range 绑定到 CUDA Profiler API： + +```shell +# --capture-range=cudaProfilerApi 让 nsys 恰好在 cudaProfilerStart/Stop 被调用时 +# 开始/停止采集；--capture-range-end=repeat 允许在一次会话中进行多次 start/stop 循环。 +nsys profile \ + --trace=cuda,nvtx,osrt \ + --capture-range=cudaProfilerApi \ + --capture-range-end=repeat \ + -o xllm_profile \ + <你的 xllm serve 启动命令> --enable_online_profile=true --profile_backend=cuda +``` + +用同样的 `/start_profile` 与 `/stop_profile` 接口控制采集窗口。当服务进程退出时，`nsys` 会将报告写入 `xllm_profile.nsys-rep`。 + +## 查看 trace + +对于 `torch` 后端，在 [Perfetto](https://ui.perfetto.dev)、`chrome://tracing` 或 TensorBoard 中打开生成的 `.pt.trace.json`。 + +对于 `cuda` 后端，在 Nsight Systems GUI 中打开生成的 `.nsys-rep`，或在命令行汇总： + +```shell +nsys stats xllm_profile.nsys-rep +``` + +## 注意事项 + +- 当前仅支持 **CUDA**。其他芯片上调用这两个接口会返回错误。 +- 仅当 `--enable_online_profile=true` 时这两个接口才会生效；否则会返回错误并提示如何开启该功能。 +- 使用 `--profile_backend=cuda` 时，`cudaProfilerStart`/`cudaProfilerStop` 只有在服务运行于 `nsys`（或 `ncu`）这类配置了 `--capture-range=cudaProfilerApi` 的 profiler 之下时才会生效。未挂载此类 profiler 时调用接口无害，但不会产生 trace。对于多进程 / 多 GPU 场景，`nsys` 推荐使用 `--trace-fork-before-exec=true` 启动，以便子 worker 进程也被采集。 +- 采集会带来运行时开销，仅建议在诊断时开启，不要在稳态生产服务中长期开启，且采集窗口应尽量短。 diff --git a/docs/src/content/docs/zh/dev_guide/tilelang_ascend_kernel_dev.md b/docs/src/content/docs/zh/dev_guide/tilelang_ascend_kernel_dev.md new file mode 100644 index 0000000000..4969469e47 --- /dev/null +++ b/docs/src/content/docs/zh/dev_guide/tilelang_ascend_kernel_dev.md @@ -0,0 +1,399 @@ +--- +title: "xLLM Ascend TileLang Kernel 开发指南" +sidebar: + order: 2 +--- +本文说明在 xLLM 中新增或修改 Ascend TileLang kernel 的开发方式。示例全程使用当前的 `rope` kernel。 + +相关目录： + +- Python kernel 定义：`xllm/xllm/compiler/tilelang/targets/ascend/kernels` +- NPU runtime wrapper：`xllm/xllm/core/kernels/npu/tilelang` + +构建和测试应在 NPU 容器中执行。 + +## 1. 先判断修改类型 + +- 新增 `specialization` + - 给现有 kernel 增加一组新的编译参数组合 + - 仍复用同一个 wrapper、同一套 runtime dispatch 字段和同一套 C ABI + - 典型动作是修改 `DISPATCH_SCHEMA` 或 `SPECIALIZATIONS` +- 新增 `kernel` + - 新增一个新的逻辑算子 + - 典型动作是新增 Python kernel 文件、wrapper C++ 文件和一条 CMake 接线 + +对 `rope` 来说： + +- 给 `SPECIALIZATIONS` 增加一项 `{"variant_key": "...", "head_dim": ..., "rope_dim": ..., "dtype": ...}`，这是新增 `specialization` +- 新增一个新的 `xxx_wrapper.cpp` 对外接口，这是新增 `kernel` + +## 2. 开发顺序 + +推荐按下面顺序开发： + +1. 在 `rope.py` 这类 Python 文件里先写 TileLang kernel 实现 +2. 实现 `generate_source(...)`，把 kernel lower 成 Ascend-C 源码 +3. 声明 `DISPATCH_SCHEMA` 和 `SPECIALIZATIONS` +4. 先生成一次 `registry.inc` 并查看内容 +5. 再写或修改 wrapper 里的 runtime specialization 构造逻辑 +6. 接入 CMake 并运行测试 + +这个顺序的重点是： + +- 先把 kernel 本身实现出来 +- 再把 runtime dispatch schema 固定下来 +- 最后根据生成出来的 `registry.inc` 写 wrapper + +## 3. 编写 Python Kernel + +以 `rope.py` 为例，Python 侧可以按三层理解： + +- `build_rope_kernel(...)`：kernel 实现 +- `generate_source(...)`：AOT 导出 +- `RopeKernel`：注册 kernel，并声明 dispatch schema 与编译实例 + +### 3.1 实现 `build_rope_kernel(...)` + +`build_rope_kernel(...)` 才是 TileLang kernel 的实现主体。这里负责写： + +- `@T.prim_func` +- 输入输出张量 shape +- `with T.Kernel(...)` 下的并行任务组织 +- UB 分配和实际计算逻辑 + +`rope.py` 的精简骨架如下： + +```python +def build_rope_kernel( + head_dim: int, + rope_dim: int, + vec_core_num: int, + ub_buffer_bytes: int, +): + task_num = vec_core_num + m_num = vec_core_num // 2 + + @T.prim_func + def rope_in_place_kernel(...): + with T.Kernel(m_num, is_npu=True) as (cid, vid): + task_id = cid * 2 + vid + ... + + return rope_in_place_kernel +``` + +这里的 `head_dim`、`rope_dim` 是这一组实现真正依赖的编译参数。 + +`rope` 这类 vector kernel 还要遵守当前 AOT 使用方式下的固定任务约定。当前路径是 AOT 编译，kernel launch 的 `block_num` 会在编译时固定下来，因此： + +- 运行时输入 shape 不影响 kernel launch 的 `block_num` +- 运行时输入 shape 只影响固定任务之间的 workload 切分 + +当前 `rope.py` 的约定是： + +```python +task_num = vec_core_num +m_num = vec_core_num // 2 + +with T.Kernel(m_num, is_npu=True) as (cid, vid): + task_id = cid * 2 + vid +``` + +这表示： + +- `cid` 范围是 `[0, vec_core_num // 2)` +- `vid` 范围是 `[0, 2)` +- 总任务数固定为 `task_num = vec_core_num` + +因此，`rope.py` 在推导单个 specialization 的编译 token 数时，也按固定任务数计算： + +```python +max_rows_num_in_ub = _derive_max_rows_num_in_ub(...) +compile_num_tokens = task_num * max_rows_num_in_ub +``` + +### 3.2 实现 `generate_source(...)` + +`generate_source(...)` 负责把上面的 TileLang kernel lower 成最终源码。导出层的职责，是把一组 specialization 参数转换成可编译的 Ascend-C 源码。 + +对 `rope` 来说，核心逻辑如下： + +```python +@staticmethod +def generate_source(head_dim: int, rope_dim: int, dtype: str) -> str: + vec_core_num = detect_vec_core_num() + tilelang_kernel = build_rope_kernel( + head_dim=head_dim, + rope_dim=rope_dim, + vec_core_num=vec_core_num, + ub_buffer_bytes=FIXED_UB_BUFFER_BYTES, + ) + with tilelang.tvm.transform.PassContext(...): + kernel = tilelang.engine.lower(tilelang_kernel) + return kernel.kernel_source +``` + +这里的规则是： + +- `generate_source(...)` 的输入来自当前这组 `SPECIALIZATIONS` +- `generate_source(...)` 内部调用 `build_rope_kernel(...)` +- 返回值是 lower 后的源码字符串 + +### 3.3 声明 `DISPATCH_SCHEMA` 与 `SPECIALIZATIONS` + +当 kernel 实现和导出层写完后，再通过 `@register_kernel` 类把它接入框架。 + +`rope.py` 当前的最小模板如下： + +```python +from ....common.spec import DispatchField, TilelangKernel, register_kernel + +@register_kernel +class RopeKernel(TilelangKernel): + DISPATCH_SCHEMA = [ + DispatchField("head_dim", "int32"), + DispatchField("rope_dim", "int32"), + DispatchField("dtype", "dtype"), + ] + SPECIALIZATIONS = [ + { + "variant_key": "hd128_rd128_bf16", + "head_dim": 128, + "rope_dim": 128, + "dtype": "bf16", + }, + { + "variant_key": "hd576_rd64_bf16", + "head_dim": 576, + "rope_dim": 64, + "dtype": "bf16", + }, + ] + + @staticmethod + def generate_source(head_dim: int, rope_dim: int, dtype: str) -> str: + ... +``` + +这里要分清楚两个概念： + +- `DISPATCH_SCHEMA` + - 定义 runtime specialization 的字段名、顺序和类型 + - 是 C++ 侧 specialization struct、builder 和查表接口的单一真相源 +- `SPECIALIZATIONS` + - 表示要实际编译出的实例集合 + - 每一项都对应一个 variant + +规则如下： + +- `DISPATCH_SCHEMA` 中每个字段都必须出现在每一项 `SPECIALIZATIONS` 里 +- `SPECIALIZATIONS` 中可以有额外字段，这些字段会传给 `generate_source(...)`，但不会进入 runtime dispatch schema +- `variant_key` 是这一组 specialization 的唯一标识 +- `DISPATCH_SCHEMA` 和 `SPECIALIZATIONS` 必须与 runtime specialization 一一对应 + +对 `rope` 来说，runtime dispatch 维度是： + +- `head_dim` +- `rope_dim` +- `dtype` + +所以这三个字段必须同时出现在： + +- `DISPATCH_SCHEMA` +- 每一项 `SPECIALIZATIONS` + +构建时，Ascend build 会根据主构建路径传入的 `--device a2|a3` 解析实际使用的 `bisheng_arch`。 + +### 3.4 查看生成的 Ascend-C 源码 + +在调试 `build_rope_kernel(...)` 的实现细节，或者比较不同 kernel 写法对最终代码生成的影响时，建议通过公共入口 `compile-kernels` 重新生成产物，再查看对应 specialization 的 Ascend-C 源码。 + +对 `rope` 来说，可以先固定： + +- `head_dim=576` +- `rope_dim=64` +- `dtype=bf16` + +然后重新生成 `rope` 的编译产物： + +```bash +python xllm/compiler/tilelang_launcher.py compile-kernels \ + --target ascend \ + --device a3 \ + --output-root /tmp/tilelang_debug \ + --kernels rope \ + --force +``` + +这里建议带上 `--force`，保证源码和 object 会按当前修改重新生成，不直接命中旧 cache。 + +这里把 `--output-root` 指到独立的调试目录 `/tmp/tilelang_debug`，这样当前命令只会在这个目录下生成 `rope` 的调试产物，不会和主构建目录里的其他 kernel 产物混在一起。 + +执行后，可以直接查看对应 specialization 的源码中的入口函数、UB 分配和向量计算逻辑： + +```bash +sed -n '1,200p' \ + /tmp/tilelang_debug/targets/ascend/rope/hd576_rd64_bf16/rope_hd576_rd64_bf16_kernel.cpp + +rg -n 'extern "C"|__global__|alloc_ub|alloc_shared|g_tilingKey' \ + /tmp/tilelang_debug/targets/ascend/rope/hd576_rd64_bf16/rope_hd576_rd64_bf16_kernel.cpp +``` + +如果要比较两种 kernel 写法的差异，做法是保持 specialization 不变，在修改前后各执行一次 `compile-kernels --force`，再对生成的 `.cpp` 做 diff： + +```bash +cp /tmp/tilelang_debug/targets/ascend/rope/hd576_rd64_bf16/rope_hd576_rd64_bf16_kernel.cpp \ + /tmp/rope_before.cpp + +diff -u /tmp/rope_before.cpp \ + /tmp/tilelang_debug/targets/ascend/rope/hd576_rd64_bf16/rope_hd576_rd64_bf16_kernel.cpp +``` + +这样可以把“specialization 变化”和“kernel 实现变化”分开看。 + +执行后，可以重点查看这些文件： + +- `/tmp/tilelang_debug/targets/ascend/rope/hd576_rd64_bf16/rope_hd576_rd64_bf16_kernel.cpp` +- `/tmp/tilelang_debug/targets/ascend/rope/registry.inc` +- `/tmp/tilelang_debug/targets/ascend/rope/manifest.json` + +这三类文件分别对应： + +- 某个 specialization 的最终 Ascend-C 源码 +- wrapper 会直接包含的 runtime dispatch 接口 +- 当前 kernel 的全部编译产物记录 + +调试顺序建议是： + +1. 先执行 `compile-kernels --force` 重新生成当前 kernel 的产物 +2. 查看对应 specialization 的 `.cpp`，分析代码生成结果 +3. 再看 `registry.inc` 和 `manifest.json` 是否符合预期 +4. 最后通过 `rope_wrapper_test` 看完整接入后的结果和性能 + +## 4. 修改 Wrapper + +新增 `kernel` 时，需要新增 wrapper。新增 `specialization` 时，只有 runtime specialization 语义变化，才需要同步修改 wrapper。 + +对 `rope_wrapper.cpp` 来说，人工需要保留的内容是： + +- tensor shape、dtype、layout 校验 +- 把输入整理成 `x_rows / sin_rows / cos_rows` +- 从 tensor 构造 runtime specialization +- 组装 launch 参数并调用 `entry->fn(...)` + +### 4.1 `registry.inc` 会自动生成什么 + +`registry.inc` 由 Python 侧的 `DISPATCH_SCHEMA`、`SPECIALIZATIONS` 和导出出来的 Ascend-C ABI 自动生成。 + +对 `rope` 来说，生成内容包括： + +- `RopeSpecialization` +- `RopeHeadDim` +- `RopeRopeDim` +- `RopeDType` +- `RopeKernelFn` +- `make_rope_specialization(...)` +- `find_rope_kernel_entry(...)` +- `available_rope_variant_keys()` + +对 `rope_wrapper.cpp` 来说，`registry.inc` 会直接提供 `RopeSpecialization`、`operator==(...)`、`RopeKernelFn` 等 dispatch 相关定义。`dtype` 转换统一使用公共函数 `to_tilelang_dtype(...)`。 + +### 4.2 wrapper 里真正要写的东西 + +`rope_wrapper.cpp` 中最关键的人工逻辑，是从 tensor 构造 runtime specialization。当前写法如下： + +```cpp +RopeSpecialization build_runtime_specialization(const torch::Tensor& x_rows) { + return make_rope_specialization( + RopeHeadDim{static_cast(x_rows.stride(0))}, + RopeRopeDim{static_cast(x_rows.size(1))}, + RopeDType{to_tilelang_dtype(x_rows.scalar_type())}); +} +``` + +对 `rope` 而言： + +- `head_dim` 对应 `x_rows.stride(0)`，也就是 kernel 使用的 `x_stride` +- `rope_dim` 对应 `x_rows.size(1)` +- `dtype` 对应 `x_rows.scalar_type()` + +运行时路径是： + +1. wrapper 把输入整理成 `x_rows / sin_rows / cos_rows` +2. `build_runtime_specialization(...)` 从 `x_rows` 构造 specialization +3. `find_rope_kernel_entry(...)` 在静态 registry 中精确匹配 +4. 命中后通过 `entry->fn(...)` 调用实际编译出来的符号 + +当前查找策略是线性扫描加精确匹配。只要 `head_dim`、`rope_dim`、`dtype` 有一个不一致，就不会命中。 + +所以新增 `specialization` 时，要重点核对的是： + +- Python 侧 `DISPATCH_SCHEMA` 的字段语义 +- Python 侧 `SPECIALIZATIONS` 的字段值 +- wrapper 里 `build_runtime_specialization(...)` 构造出的字段值 + +这三者必须完全对齐。 + +### 4.3 先生成并查看 `registry.inc` + +在写或修改 wrapper 之前，先生成一次 `registry.inc` 并查看内容。重点看： + +- 生成出来的 `RopeSpecialization` 字段顺序是否符合预期 +- 生成出来的字段包装类型名是否符合预期 +- `make_rope_specialization(...)` 的参数顺序是什么 +- 生成出来的 entry symbol 名称是什么 + +`registry.inc` 是 wrapper 的直接契约，先看它，再写 wrapper。 + +## 5. 修改 CMake + +新增 `kernel` 时，在 `xllm/xllm/core/kernels/npu/tilelang/CMakeLists.txt` 中接入这个 kernel。 + +CMake 接入统一通过高层 helper 完成： + +- `tilelang_register_runtime_kernel(NAME WRAPPER_SRCS )` + +以 `rope` 为例，最小模板如下： + +```cmake +tilelang_register_runtime_kernel( + NAME rope + WRAPPER_SRCS rope_wrapper.cpp +) +``` + +这条 helper 会完成： + +- 按 `TILELANG_GENERATED_ROOT/targets/ascend//manifest.json` 推导 manifest 路径 +- 导入 manifest +- 把该 kernel 的 wrapper source 和 compiled objects 加入 `tilelang_kernels` +- 自动追加 `XLLM_TL__REGISTRY_INC=...` compile definition + +因此，新增一个 runtime kernel 时，CMake 侧主要就是两件事： + +1. 保证 Python 侧已经能生成该 kernel 的 manifest +2. 在 `tilelang` 的 CMakeLists 里新增一条 `tilelang_register_runtime_kernel(...)` + +日常新增 kernel 时，直接在 CMake 中增加一条 `tilelang_register_runtime_kernel(...)`。`tilelang_import_kernel_manifest(...)` 作为这条高层 helper 的实现基础，保留在底层。 + +## 6. 验证 + +推荐按下面顺序验证： + +1. 先编译 TileLang kernel，并查看生成的 `registry.inc` +2. 再跑完整的 wrapper 测试 + +常用命令： + +```bash +python xllm/compiler/tilelang_launcher.py compile-kernels \ + --target ascend \ + --device a3 \ + --output-root build/cmake.linux-aarch64-cpython-311/xllm/compiler/tilelang \ + --kernels rope + +python setup.py test --test-name rope_wrapper_test --device a3 +``` + +第一条命令用于生成 `manifest.json`、`registry.inc` 和 object；第二条命令用于验证完整接入路径。 + diff --git a/docs/src/content/docs/zh/features/async_schedule.md b/docs/src/content/docs/zh/features/async_schedule.md new file mode 100644 index 0000000000..7f50927b7f --- /dev/null +++ b/docs/src/content/docs/zh/features/async_schedule.md @@ -0,0 +1,33 @@ +--- +title: "异步调度" +sidebar: + order: 10 +--- +## 背景 +大模型推理过程可划分为3个阶段，包括CPU执行调度准备模型输入阶段，device计算阶段，CPU处理输出阶段。 +由于解码操作的序列性，step-i+1 的输入需要依赖 step-i 的输出结果， +上述3个阶段需要按顺序串行执行，导致在CPU执行阶段1和3的时候，device侧空闲等待出现空泡，资源利用不充分。 + +## 功能介绍 + +xLLM在框架层支持了异步调度功能，在device执行 step-i 计算的同时提前让CPU执行 step-i+1 的调度操作，device在完成 step-i 计算后可立即开始 step-i+1 的计算，从而消除空泡。 +具体地，CPU在发起 step-i 计算调用后，不等待device计算完成，为 step-i 的请求构造fake token，使用fake token执行 step-i+1 的调度操作，分配KV Cache等；device在启动 step-i+1 的计算时，用 step-i 计算出来的true token替换fake token，保证计算的正确性。CPU在另外的线程中同步处理 step-i 的结果返回给client。 + +整体架构如图，实现中CPU侧执行阶段1和阶段3的操作分别采用了不同的线程池，rpc等函数调用采用C++ future和promise非阻塞调用，实现全异步runtime。![异步调度](../../assets/async_schedule_architecture.jpg) + +## 使用方式 + +xLLM中提供了gflags参数`enable_schedule_overlap`，默认false，如需开启在xLLM的服务启动脚本中设置为true即可，示例如下： +```shell +--enable_schedule_overlap=true +``` + +## 性能效果 +- 异步调度开启后，两个step之间的device空闲时在200us左右，基本类似一个kernel launch的时间。 +- 在DeepSeek-R1-Distill-Qwen-1.5B模型上，限制TPOT 50ms，吞吐 **提升17%**。 + +:::caution[注意] +- 异步调度功能会在服务端额外计算一个step，当使用场景中输出token数量较少，或是类似embedding模型只一次性输出的场景，会影响服务端吞吐，所以强制关闭异步调度。 +- VLM模型正在适配中，暂时会强制关闭异步调度。 + +::: diff --git a/docs/src/content/docs/zh/features/chunked_scheduler.md b/docs/src/content/docs/zh/features/chunked_scheduler.md new file mode 100644 index 0000000000..7cd595248e --- /dev/null +++ b/docs/src/content/docs/zh/features/chunked_scheduler.md @@ -0,0 +1,21 @@ +--- +title: "ChunkedPrefill调度器" +sidebar: + order: 21 +--- +## 功能介绍 +xLLM支持chunked prefill调度策略。Chunked prefill是一种优化大语言模型推理的技术，将长prompt分割成多个较小的chunk进行分批处理，而不是一次性处理整个prompt。 +这种方法可以有效降低显存峰值使用量，提高Device利用率，并且能够更好地与decode阶段的请求进行调度和混合处理。 + +## 使用方式 +上述策略已在xLLM实现，并向外暴露gflag参数，控制功能的开关。 + +- 开启chunked prefill，并设置chunked_size，如果不手动设置chunked size，则默认等于max_tokens_per_batch。 +```bash +--enable_chunked_prefill=true +--max_tokens_per_chunk_for_prefill=20480 # optional +``` + +## 性能效果 +开启chunked_prefill之后，在Qwen3-8B模型上，限制TPOT 50ms，TTFT时延 **下降46%**。 + diff --git a/docs/src/content/docs/zh/features/disagg_pd.md b/docs/src/content/docs/zh/features/disagg_pd.md new file mode 100644 index 0000000000..4dbdd6b310 --- /dev/null +++ b/docs/src/content/docs/zh/features/disagg_pd.md @@ -0,0 +1,79 @@ +--- +title: "PD分离" +sidebar: + order: 30 +--- +## 背景 +LLM在线推理服务通常需要满足TTFT和TPOT两项性能指标，而传统的Contiguous Batching调度策略将Prefill和Decode请求混合在一起调度，导致P和D会互相抢占计算资源，影响性能指标无法最大程度的利用计算资源。为解决上述矛盾，将Prefill和Decode两阶段拆分到独立的计算资源并行执行，从而同时降低TTFT和TPOT并提升吞吐量。 +## 功能介绍 +xLLM PD分离功能主要通过以下三个模块实现： + +- **etcd**: 存储实例信息等元数据 +- **xLLM Service**: 调度请求和管理所有计算实例 +- **xLLM**: 请求计算实例 + +整体架构图如下： +![xLLM PD分离架构图](../../assets/pd_architecture.jpg) +## 功能使用示例 +### 使用准备 +#### 安装相关依赖 +- **xLLM**: 参见[安装编译](/zh/getting_started/quick_start/) +- **xLLM Service**: 参见[xLLM Service](https://github.com/xLLM-AI/xllm-service) + +### 启动PD分离服务 +1. 启动etcd +``` +./etcd +``` +2. 启动xLLM Service +```bash +ENABLE_DECODE_RESPONSE_TO_SERVICE=true ./xllm_master_serving --etcd_addr="127.0.0.1:12389" --http_server_port 28888 --rpc_server_port 28889 --tokenizer_path=/path/to/tokenizer_config_dir/ +``` +3. 启动xLLM + +4. 以Qwen2-7B为例 + - 启动Prefill实例 + ```bash + /path/to/xllm --model=Qwen2-7B-Instruct \ + --port=8010 \ + --devices="npu:0" \ + --master_node_addr="127.0.0.1:18888" \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_disagg_pd=true \ + --instance_role=PREFILL \ + --etcd_addr=127.0.0.1:12389 \ + --transfer_listen_port=26000 \ + --disagg_pd_port=7777 \ + --node_rank=0 \ + --nnodes=1 + ``` + - 启动Decode实例 + ```bash + /path/to/xllm --model=Qwen2-7B-Instruct \ + --port=8020 \ + --devices="npu:1" \ + --master_node_addr="127.0.0.1:18898" \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_disagg_pd=true \ + --instance_role=DECODE \ + --etcd_addr=127.0.0.1:12389 \ + --transfer_listen_port=26100 \ + --disagg_pd_port=7787 \ + --node_rank=0 \ + --nnodes=1 + ``` + 需要注意： + + - PD分离需要读取`/etc/hccn.conf`文件，确保将物理机上的该文件映射到了容器中 + + - `etcd_addr`需与`xllm_service`的`etcd_addr`相同 + +:::tip[Chunked prefill的Prefill实例支持prefix cache] +使用chunked-prefill PD调度器时，Prefill实例已支持prefix cache。开启后，调度器会先匹配已有prefix cache block，再计算当前chunk budget，避免重复计算已缓存的prompt block。 +``` shell +--enable_chunked_prefill=true +--enable_prefix_cache=true +``` +::: diff --git a/docs/src/content/docs/zh/features/eplb.md b/docs/src/content/docs/zh/features/eplb.md new file mode 100644 index 0000000000..6c81be2500 --- /dev/null +++ b/docs/src/content/docs/zh/features/eplb.md @@ -0,0 +1,39 @@ +--- +title: "MoE负载均衡（EPLB）" +sidebar: + order: 71 +--- +## 背景介绍 + +MoE模型依赖动态路由分配tokens给专家，但实际部署中因数据分布不均，导致专家负载失衡（部分过载、部分闲置）。专家冗余调整（如新增/删除副本）需要消耗额外显存，并可能因权重迁移影响推理延迟，如何高效、平滑地完成是一大挑战。为此，采用专家冗余策略（复制热点专家）结合分层和全局动态负载均衡实现了动态的MOE负载均衡。 + +## 功能介绍 +xLLM MoE负载均衡（EPLB）功能主要通过以下三个模块实现： +- eplb manager: 负责专家负载并收集并管理专家分布更新更新，采用逐层更新机制，根据专家负载变化情况判断是否更新该层。 +- eplb excutor: 实际专家分布更新执行器。 +- eplb policy: 新专家负载表生成策略。 +整体架构图如下： +![xLLM eplb](../../assets/eplb_architecture.png) + +## 使用方式 +只需在启动 xLLM 时加上下面的 gflag 参数即可： +替换为实际的Device个数 ep_size要与device个数保持一致 + +- xLLM中提供了gflags参数`enable_eplb`，默认false，如需开启动态专家负载均衡，在xLLM的服务启动脚本中设置为true即可。 +- `expert_parallel_degree`与`ep_size`为moe相关参数，`expert_parallel_degree`需要设置为`2`，`ep_size`要与实际NPU/GPU卡个数保持一致。参考 [moe_params](/zh/features/moe_params/) +- `eplb_update_interval`为专家分布更新时间间隔，单位为妙，默认值为1000. +- 专家分布更新采用根据专家负载的逐层更新机制，当某一层专家的前后两次的负载相似度小于`eplb_update_interval`时选择更新该层，默认值为1，取之范围为(0,1)。 + +```bash + --enable_eplb=true + --expert_parallel_degree=2 + --ep_size=16 + --eplb_update_interval=2000 + --eplb_update_threshold=0.9 +``` + +## 未来工作 + * 采用更加细粒度的专家更新机制。 + + * 与调度层结合，通过请求batch的重组实现更好的负载均衡。 + diff --git a/docs/src/content/docs/zh/features/flashcomm.md b/docs/src/content/docs/zh/features/flashcomm.md new file mode 100644 index 0000000000..8556773ca4 --- /dev/null +++ b/docs/src/content/docs/zh/features/flashcomm.md @@ -0,0 +1,124 @@ +--- +title: "FlashComm" +sidebar: + order: 82 +--- + +## 功能介绍 + +FlashComm 是 xLLM 在 NPU Tensor Parallel 推理场景下的 prefill 通信优化特性。它的目标是减少长输入 prefill 阶段中 row-parallel 线性层后的通信开销，并在合适场景下使用 Matmul + ReduceScatter 融合算子降低 kernel launch 和通信调度成本。 + +当前 FlashComm 主要包含两部分： + +- **序列维度分片**：在 prefill 阶段将 token 序列按 TP rank 切分，让后续部分计算在本 rank 的 token shard 上执行。 +- **MMRS 融合算子**：在 row-parallel 线性层中，将普通 `matmul + reduce_scatter` 替换为 torch_npu 的 `npu_mm_reduce_scatter_base`，即 Matmul + ReduceScatter 融合路径。 + +FlashComm 默认关闭。即使打开总开关，也只有满足运行条件时才会真正启用；不满足条件时会走原有执行路径。 + +## 设计说明 + +FlashComm 的执行流程如下： + +1. 请求进入 prefill 阶段后，运行时根据 token 数、并行配置和开关构造 FlashComm 上下文。 +2. 当上下文生效时，输入 hidden states 会按序列维度切分到不同 TP rank。 +3. 在支持的 row-parallel 线性层中，优先尝试 MMRS 融合路径。 +4. 如果 MMRS 不适用，例如 shape、dtype、bias 或通信上下文不满足要求，则回退到普通 matmul 和 reduce_scatter 路径。 +5. 在需要完整 hidden states 的边界处，再通过 gather 恢复完整序列。 + +当前 MMRS 路径使用 torch_npu 提供的 `npu_mm_reduce_scatter_base`。xLLM 侧只保留薄封装，用于完成输入校验、HCCL group 获取、`comm_mode` 选择和日志记录，不重新实现 Matmul + ReduceScatter kernel。 + +## 适用场景 + +FlashComm 更适合以下场景： + +- NPU 后端。 +- 长输入 prefill，例如输入长度大于等于 8K tokens。 +- TP 较大，当前默认建议 `TP >= 8`。 +- `dp=1` 且 `cp=1`。 +- prefill 占端到端时延比例较高，例如 8K/128、32K/1K 等长 prompt 场景。 +- 使用 BF16/FP16 的非量化 row-parallel 线性层。 + +FlashComm 不适合或收益有限的场景： + +- decode 阶段。FlashComm 只优化 prefill，不优化 decode，因此 TPOT 通常不会直接受益。 +- 短输入，例如 2K 输入场景。通信占比不足时，切分、gather 和调度开销可能抵消收益。 +- 高 decode 占比场景，例如长输出请求。整体 latency 可能主要由 decode 决定。 +- `TP < 8`、`dp > 1` 或 `cp > 1` 的场景，当前默认不会启用。 +- 量化 row-parallel 路径。当前 MMRS 只接入普通 BF16/FP16 matmul 路径。 + +## 使用方式 + +FlashComm 和 MMRS 融合算子都默认关闭。推荐在长 prefill、TP=8 或更大 TP 的 NPU 服务中显式开启： + +```bash +--enable_flashcomm1=true \ +--enable_mmrs_fusion=true \ +--flashcomm1_min_prefill_tokens=8192 \ +--mmrs_comm_mode=aiv +``` + +推荐同时开启 Graph Mode，降低 Host 侧调度开销： + +```bash +--enable_graph=true \ +--enable_prefill_piecewise_graph=true +``` + +完整推荐配置示例： + +```bash +--enable_graph=true \ +--enable_prefill_piecewise_graph=true \ +--enable_flashcomm1=true \ +--enable_mmrs_fusion=true \ +--flashcomm1_min_prefill_tokens=8192 \ +--mmrs_comm_mode=aiv +``` + +参数说明： + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `enable_flashcomm1` | `false` | FlashComm 总开关 | +| `enable_mmrs_fusion` | `false` | 是否启用 Matmul + ReduceScatter 融合算子 | +| `flashcomm1_min_prefill_tokens` | `8192` | prefill token 数达到该阈值后才允许启用 FlashComm | +| `mmrs_comm_mode` | `aiv` | torch_npu MMRS 通信模式，可选 `aiv`、`ai_cpu`、`none` | + +通常建议保持 `mmrs_comm_mode=aiv`。如果某些 shape 在 AIV 路径出现 AICore 异常，可以临时切换为： + +```bash +--mmrs_comm_mode=ai_cpu +``` + +## 最优配置建议 + +推荐从以下配置开始评估： + +| 场景 | 建议 | +|------|------| +| 8K 输入 / 短输出 | 推荐开启 FlashComm 和 MMRS | +| 32K 输入 / 中短输出 | 推荐开启 FlashComm 和 MMRS | +| 2K 输入 / 长输出 | 不建议默认开启，收益通常不稳定 | +| TP=2 或 TP=4 | 不建议默认开启 | +| TP=8 | 当前最推荐评估的配置 | +| Chunked Prefill | 可以开启，但建议 chunk size 不小于 `flashcomm1_min_prefill_tokens`，否则单个 chunk 可能不会触发 FlashComm | + +如果业务 workload 混合了短输入和长输入，建议保持默认关闭，并只在长输入服务、长上下文模型或独立部署的长 prompt workload 中开启。 + +## 性能与正确性注意事项 + +- FlashComm 只优化 prefill，因此观察收益时应重点关注 TTFT、prefill throughput 和 profiling 中 prefill 阶段的通信变化。 +- TPOT、decode throughput 和长输出 latency 不一定改善；如果 decode 占比高，端到端 latency 可能看不到明显收益。 +- 开启 MMRS 后应确认 profiling 中部分 row-parallel 后的 `allReduce` 或 `reduce_scatter` 开销被 Matmul + ReduceScatter 融合路径替代。 +- 如果看到额外 gather、layout 转换或 Host 调度开销增加，可能会抵消 MMRS 的收益。 +- 建议使用 warmup 后的多轮稳定请求评估，不要直接使用 profiling run 的绝对时延作为性能结论。 + +## 验证建议 + +上线或调参前，建议至少完成以下验证： + +1. 对比 `enable_flashcomm1=false, enable_mmrs_fusion=false` 和 `enable_flashcomm1=true, enable_mmrs_fusion=true`。 +2. 使用相同模型、相同 TP、相同输入输出长度和相同并发。 +3. 记录 TTFT、TPOT、prompt throughput、decode throughput、request throughput 和 latency。 +4. 对长输入场景额外采集 profiling，确认 MMRS 路径真正命中。 +5. 做小规模数值一致性检查，确认开启和关闭 FlashComm 的输出一致。 diff --git a/docs/src/content/docs/zh/features/global_kvcache.md b/docs/src/content/docs/zh/features/global_kvcache.md new file mode 100644 index 0000000000..fb4ffd8156 --- /dev/null +++ b/docs/src/content/docs/zh/features/global_kvcache.md @@ -0,0 +1,38 @@ +--- +title: "全局多级KV Cache" +sidebar: + order: 51 +--- +## 背景 +大型语言模型（LLM）解码阶段因自回归生成需频繁访问历史KV缓存，导致显存带宽成为瓶颈。随着模型规模与上下文窗口扩大（如128K Token消耗超40GB显存），单卡显存压力剧增。现有方案（如vLLM）在长上下文场景下存在明显局限：预填充耗时激增、解码阶段显存带宽争抢严重，为满足SLO（TTFT<2s, TBT<100ms）常需过量预留资源，致使GPU利用率不足40%，且难以利用跨服务器资源。为此，我们提出分布式全局多级KV缓存管理系统，采用存算一体架构以突破单机资源限制。 + +## 功能介绍 +xLLM 全局KV Cache功能主要通过以下三个模块实现： +- etcd: 集群服务注册、负载信息同步及全局缓存状态管理 +- xLLM Service: 调度请求和管理所有计算实例 +- xLLM: 请求计算实例 + +整体架构图如下： +![xLLM 全局多级KV Cache](../../assets/globalkvcache_architecture.png) +## 功能使用示例 +### 使用准备 +#### 安装相关依赖 +- **xLLM**: 参见[快速开始](/zh/getting_started/quick_start/) +- **xLLM Service**: 参见[xLLM Service](https://github.com/xLLM-AI/xllm-service) + +### 使用方式 +1. etcd启动配置： +```bash +./etcd --listen-peer-urls=http://0.0.0.0:10999 --listen-client-urls=http://0.0.0.0:10998 +``` +2. xLLM Service启动配置： +```bash +./xllm_master_serving --etcd_addr="127.0.0.1:10998" --http_server_port 28888 --rpc_server_port 28889 --tokenizer_path=/path/to/tokenizer_config_dir/ +``` +3. xLLM启动添加上下面的 gflag 参数即可： +```bash +--enable_service_routing=true +--enable_cache_upload=true +# PD分离暂时不支持全局KVCache管理 +--enable_disagg_pd=false +``` diff --git a/docs/src/content/docs/zh/features/graph_mode.md b/docs/src/content/docs/zh/features/graph_mode.md new file mode 100644 index 0000000000..4b0d1f098c --- /dev/null +++ b/docs/src/content/docs/zh/features/graph_mode.md @@ -0,0 +1,79 @@ +--- +title: "Graph Mode" +sidebar: + order: 81 +--- +## 概述 + +xLLM 支持 Graph Mode，通过预捕获计算图并在后续执行中重放，减少 CPU 开销并提高推理性能。Graph Mode 在不同硬件平台上均有对应实现。 + +## 功能介绍 + +为了优化 Host 侧调度性能，图模式通过在 CPU 一次提交大任务后，设备内部流式执行小 kernel，显著降低启动时间和设备气泡。 + +在 xLLM 引擎中，Graph Mode 实现了以下特性： + +### 动态维度参数化 + - 将除 num_tokens 以外的关键动态维度作为整图输入参数，包括 batch_size、kv_seq_lens、q_seq_lens、block_table_size 等，从而提高灵活性。在进行图的内存分配和内核配置时，利用这些动态参数计算实际所需值。在图启动阶段，将上述实际参数传入，以确保 kernel 能够使用正确的 stride 访问数据。 + +### Piecewise Graph + - 当部分算子不支持 graph 导致整图无法捕获（break graph）时，对 break 之后的各段（piece）分别捕获 graph。这样在无法整图捕获的情况下，仍能尽可能获得 graph mode 的收益，常用于 prefill、chunked_prefill 等场景。 + +### 多 shape 复用的显存池 + - 为了避免不同 shape 的 graph capture 分别占用独立显存，我们让不同 capture 使用不同虚拟地址空间，并共享同一组底层物理内存；同时，输入 tensor 通过持久化 buffer 与 slice 方式复用。 + +## 使用方式 + +上述功能已在 xLLM 引擎内部实现，通常通过 gflags 参数控制。 + +最小配置只需要开启 `enable_graph`，用于打开 decode 阶段的 Graph Mode： + +```shell +--enable_graph=true +``` + +常见的配套开关包括： + +- `enable_graph`：开启 decode 阶段的 Graph Mode 基础能力 +- `enable_prefill_piecewise_graph`：开启 prefill 阶段的 Piecewise Graph +- `enable_graph_mode_decode_no_padding`：decode 阶段按实际 `num_tokens` 建图，而不是按 padding 后的 shape 建图 +- `max_tokens_for_graph_mode`：限制 Graph Mode 覆盖的最大 token 数；`0` 表示不限制 + +如果希望同时开启 decode Graph 和 prefill Piecewise Graph，示例如下： + +```shell +--enable_graph=true \ +--enable_prefill_piecewise_graph=true \ +--max_tokens_for_graph_mode=2048 +``` + +如果需要在 decode 阶段启用无 padding 建图，可额外开启： + +```shell +--enable_graph=true \ +--enable_graph_mode_decode_no_padding=true +``` + +更完整的参数说明可参考 [CLI 参数说明](/zh/cli_reference/)。 + +## 性能效果 + +- 开启 Graph Mode 后，在 Qwen3-0.6B 和 Qwen3-1.7B 等模型上，decode 阶段吞吐 **提升约 8%–10%**。 + +## 模型支持 + +下表列出目前各模型在 ACLGraph、CudaGraph、MLUGraph 上的支持情况。 + +| 模型 | ACLGraph | CudaGraph | MLUGraph | +|------|----------|-----------|----------| +| Qwen3/Qwen3-MoE | ✅ | ✅ | ✅ | +| DeepseekV3.2 | ✅ | | ✅ | +| GLM4.5/4.6/4.7 | ✅ | | | +| Qwen2.5-VL | | | ✅ | +| Qwen3-VL/Qwen3-VL-MoE | ✅ | | | +| GLM4V | ✅ | | | +| GLM4V-MoE | ✅ | | | + +## 相关文档 +- 更详细的 Graph Mode 设计与实现说明（含 ACL Graph / CUDA Graph 基本原理、动态维度参数化、Piecewise Graph 与多 shape 复用内存方案）见：[Graph Mode 设计文档](/zh/design/graph_mode_design/) + diff --git a/docs/src/content/docs/zh/features/moe_params.md b/docs/src/content/docs/zh/features/moe_params.md new file mode 100644 index 0000000000..dc31367cca --- /dev/null +++ b/docs/src/content/docs/zh/features/moe_params.md @@ -0,0 +1,22 @@ +--- +title: "EP并行" +sidebar: + order: 70 +--- +## 背景介绍 +在部署DeepSeek-R1 671B参数规模模型时传统分布式部署面临、显存利用率低、通信开销大、硬件成本高昂等核心瓶颈，因此需要引入ep并行。 ++ 在同等资源下，单张卡上的Expert越少，可用于KV Cache的显存越多，可Cache的token个数越多。 ++ 因MLA的特性，同等资源下TP Size越小，冗余的KV Cache就越少，可Cache的token个数越多。 ++ 采用大规模ep并行部署，可以将同一个expert的token计算集中到同一设备上，提高硬件利用率 +## 参数设置 ++ dp_size：设置Attention部分的dp规模大小，默认值为1，可设置为2的指数倍，当dp_size不等于卡数时，dp组内为tp并行. ++ ep_size：设置MoE部分的ep规模大小，默认值为1，可设置为2的指数倍，当ep_size不等于卡数时，dp组内为tp并行. ++ expert_parallel_degree ：ep并行相关参数，不开启ep时默认设置为0，开启ep时默认为1，此时为ep level1，当ep_size等于卡数时可以设置为2开启ep level2. + +支持 MLA 的模型会自动开启 MLA，不再需要手动配置。 +## 方案设计 ++ 当开启ep时，默认为ep level1，此时attn与moe部分计算完成后，通过All Gather全卡通讯将数据发送到下一阶段，以64卡attn部分dp32tp2 moe部分ep32tp2为例，执行流程如下： +![Alt text](../../assets/moe_eplevel1.jpg) ++ 当ep_size设置为卡数时，可以开启ep level2，此时attn部分与moe部分之间通讯变为ALL2ALL，只向需要的卡发送数据，降低通讯量与通讯开销，以64卡部署为例，执行流程如下： +![Alt text](../../assets/moe_eplevel2.jpg) + diff --git a/docs/src/content/docs/zh/features/mtp.md b/docs/src/content/docs/zh/features/mtp.md new file mode 100644 index 0000000000..eb06c21898 --- /dev/null +++ b/docs/src/content/docs/zh/features/mtp.md @@ -0,0 +1,154 @@ +--- +title: "MTP投机推理" +sidebar: + order: 80 +--- +## 背景 +MTP是一种创新的推理阶段加速技术，专注于解决大语言模型生成过程中的效率瓶颈。MTP的本质是通过预训练阶段的特殊设计，为推理阶段提供高效的草稿token预测能力，从而显著提升模型的生成速度。其核心价值在于平衡推理效率与输出质量，为大语言模型的长序列生成问题提供了一种高效的解决方案，最终实现推理性能的优化。 + +## 功能介绍 +MTP在推理加速方面具有以下核心功能： + +- **高效草稿生成**：使用低成本的MTP结构快速生成草稿token，这些草稿token作为主模型验证的基础，大幅减少了传统自回归生成的计算开销。 + +- **批量验证机制**：主模型能够同时批量验证多个MTP生成的草稿token，而不必逐个生成和验证，显著提升了推理速度。 + +- **高采样准确率**：MTP解决了Eagle、Medusa等现有推理加速方法中的关键痛点——训练后生成的draft模块token采样率低的问题。由于MTP在预训练阶段就优化了草稿生成能力，其生成的token具有更高的准确率，减少了主模型的验证负担。 + +- **推理延迟降低**：通过预先生成多个可能的后续token，MTP有效降低了模型生成长文本时的累积延迟，使用户体验更加流畅。 + +- **资源消耗优化**：相比其他推理加速技术，MTP在保持加速效果的同时，对计算资源的额外需求更少，适合在资源受限环境下部署。 + +MTP技术为大语言模型的推理阶段提供了一种全新的效率优化方案，特别适合需要快速响应的实时应用场景，代表了语言模型推理优化的重要发展方向。 + +:::note[模型支持] +目前支持以下模型的MTP结构导出： +- DeepSeek-V3 (输入 model_type: deepseek_v3, 导出 MTP model_type: deepseek_v3_mtp) +- DeepSeek-V3.2 (输入 model_type: deepseek_v3, 导出 MTP model_type: deepseek_v32_mtp) +- DeepSeek-R1 (输入 model_type: deepseek_v3, 导出 MTP model_type: deepseek_v3_mtp) +- GLM4 MoE (如 GLM-4.5-Air, 导出 MTP model_type: glm4_moe_mtp) + +注意： +- DeepSeek V3 和 R1 的输入 model_type 都是 "deepseek_v3"，导出的 MTP 模型 model_type 为 "deepseek_v3_mtp" +- DeepSeek V3.2 的输入 model_type 是 "deepseek_v3"（但可通过 index_head_dim 等字段自动识别），导出的 MTP 模型 model_type 为 "deepseek_v32_mtp" + +::: +## 使用示例 + +### 导出模型 + +脚本会自动检测模型类型，也可以手动指定。 + +#### DeepSeek-V3 +```bash +python3 tools/export_mtp.py \ + --input-dir /path/to/DeepSeek-V3 \ + --output-dir /path/to/DeepSeek-V3-mtp +``` + +#### DeepSeek-V3.2 +```bash +python3 tools/export_mtp.py \ + --input-dir /path/to/DeepSeek-V3.2 \ + --output-dir /path/to/DeepSeek-V3.2-mtp +``` + +#### DeepSeek-R1 +```bash +python3 tools/export_mtp.py \ + --input-dir /path/to/DeepSeek-R1 \ + --output-dir /path/to/DeepSeek-R1-mtp +``` + +#### GLM4 MoE +```bash +python3 tools/export_mtp.py \ + --input-dir /path/to/GLM-4.5-Air \ + --output-dir /path/to/GLM-4.5-Air-mtp +``` + +#### 手动指定模型类型 +如果自动检测失败，可以手动指定模型类型： +```bash +python3 tools/export_mtp.py \ + --input-dir /path/to/model \ + --output-dir /path/to/model-mtp \ + --model-type deepseek_v3 # 可选: deepseek_v3 (用于V3/R1), deepseek_v32 (用于V3.2), glm4_moe +``` + +输入模型参考: +- [DeepSeek-V3](https://huggingface.co/deepseek-ai/DeepSeek-V3) +- [DeepSeek-V3.2](https://huggingface.co/deepseek-ai/DeepSeek-V3.2) +- [DeepSeek-R1](https://huggingface.co/deepseek-ai/DeepSeek-R1) +- [GLM-4.5-Air](https://huggingface.co/zai-org/GLM-4.5-Air) + +### 启动脚本 + +使用MTP进行推理时，需要同时指定主模型和草稿模型（MTP模型）。 + +#### DeepSeek-V3/V3.2/R1 启动示例 +```bash +MODEL_PATH="/models/DeepSeek-V3" +DRAFT_MODEL_PATH="/models/DeepSeek-V3-mtp" +MASTER_NODE_ADDR="127.0.0.1:42123" +START_PORT=13222 +START_DEVICE=0 +LOG_DIR="log" +NNODES=16 + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + nohup ./xllm \ + --model $MODEL_PATH \ + --devices="npu:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --draft_model $DRAFT_MODEL_PATH \ + --draft_devices="npu:$DEVICE" \ + --num_speculative_tokens 1 \ + --max_memory_utilization=0.90 \ + --max_tokens_per_batch=10000 \ + --max_seqs_per_batch=256 \ + --block_size=128 \ + --ep_size=1 \ + --dp_size=1 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --node_rank=$i > $LOG_FILE 2>&1 & + sleep 0.5 +done +``` + +#### GLM4 MoE 启动示例 +```bash +MODEL_PATH="/models/GLM-4.5-Air" +DRAFT_MODEL_PATH="/models/GLM-4.5-Air-mtp" +# ... 其他配置相同 +``` + +## 性能数据 +基于sharegpt数据集，输入长度2500，输出长度1500，请求总数80。 + +| method | Concurrency | Mean TPOT(ms) | Mean TTFT(ms) | Output Tokens/s | Total Tokens/s | +|:---------:|:-----------:|:-------------:|:-------------:|:---------------:|:--------------:| +| baseline | 1 | 40.61 | 141.80 | 24.20 | 65.77 | +| mtp | 1 | 28.33 | 142.35 | 35.19 | 95.52 | +| baseline | 2 | 42.69 | 178.59 | 45.16 | 122.74 | +| mtp | 2 | 29.81 | 187.97 | 64.75 | 175.78 | +| baseline | 4 | 46.18 | 172.34 | 79.83 | 216.96 | +| mtp | 4 | 33.54 | 194.22 | 111.18 | 301.81 | +| baseline | 8 | 53.16 | 181.49 | 110.68 | 300.81 | +| mtp | 8 | 40.99 | 203.37 | 154.46 | 419.34 | +| baseline | 16 | 68.50 | 213.89 | 143.81 | 390.84 | +| mtp | 16 | 57.04 | 254.99 | 201.89 | 548.04 | +| baseline | 20 | 74.72 | 228.80 | 154.77 | 420.65 | +| mtp | 20 | 61.73 | 264.34 | 206.24 | 559.84 | +| baseline | 40 | 119.68 | 559.32 | 180.22 | 489.80 | +| mtp | 40 | 105.70 | 544.54 | 252.91 | 686.74 | +| baseline | 80 | 180.89 | 2996.21 | 192.09 | 522.06 | +| mtp | 80 | 152.19 | 2163.72 | 278.07 | 755.12 | + diff --git a/docs/src/content/docs/zh/features/multi_streams.md b/docs/src/content/docs/zh/features/multi_streams.md new file mode 100644 index 0000000000..26ffa8a709 --- /dev/null +++ b/docs/src/content/docs/zh/features/multi_streams.md @@ -0,0 +1,29 @@ +--- +title: "多流并行" +sidebar: + order: 11 +--- +## 背景 +大模型分布式推理场景中需要引入额外的通信操作，将不同设备上的计算结果聚合在一起。以Deepseek这类大规模的MoE模型为例，分布式规模通常较大，通信开销也会随之变大。计算和通信都采用同一个stream的话，在通信的同时，device计算资源会出现浪费，一直等待通信完成才能开始后面的计算。 + +## 功能介绍 +xLLM在模型图层支持了多流并行功能，将输入的batch拆分成2个micro batches，一个流执行一个micro batch的计算操作，另一个流执行另一个micro batch的通信操作，计算和通信同时执行，从而掩盖通信开销。 +![异步调度](../../assets/multi_streams_architecture.jpg) + +## 使用方式 + +xLLM中提供了gflags参数`enable_multi_stream_parallel`，默认false，如需开启在xLLM的服务启动脚本中设置为true即可，示例如下： +```shell +--enable_multi_stream_parallel=true +``` + +## 性能效果 +prefill双流并行开启后，基本可掩盖75以上的通信开销，在DeepSeek-R1模型上，只输出1个token的情况下 + +- TTFT下降 **7%** +- 吞吐 **提升7%** + +:::caution[注意] +双流并行目前只支持prefill阶段，请求输入越长，收益越大。 +目前仅支持DeepSeek、Qwen3 dense（非MoE）模型。 +::: diff --git a/docs/src/content/docs/zh/features/multimodal.md b/docs/src/content/docs/zh/features/multimodal.md new file mode 100755 index 0000000000..01888114c3 --- /dev/null +++ b/docs/src/content/docs/zh/features/multimodal.md @@ -0,0 +1,23 @@ +--- +title: "多模态支持" +sidebar: + order: 60 +--- +本文档主要介绍xLLM推理引擎中多模态的支持进展，包括支持模型及模态类型，以及离在线接口等。 + +## 支持模型 +- Qwen2.5-VL: 包括7B/32B/72B。 +- Qwen3-VL: 包括2B/4B/8B/32B。 +- Qwen3-VL-MoE: 包括A3B/A22B。 +- MiniCPM-V-2_6: 7B。 + +## 模态类型 +- 图片: 支持单图、多图的输入，以及图片+Prompt组合、纯文本Promot等输入方式。 + +:::caution[注意事项] +- 目前多模态后端不支持prefix cache以及chunk prefill，正在支持中。 +- 目前，xLLM统一基于JinJa渲染ChatTemplate，部署MiniCPM-V-2_6，模型目录需提供ChatTemplate文件。 +- 图片支持Base64输入以及图片Url。 +- 目前多模态模型主要支持了图片模态，视频、音频等模态正在推进中。 + +::: diff --git a/docs/src/content/docs/zh/features/prefix_cache.md b/docs/src/content/docs/zh/features/prefix_cache.md new file mode 100644 index 0000000000..1767252f9c --- /dev/null +++ b/docs/src/content/docs/zh/features/prefix_cache.md @@ -0,0 +1,36 @@ +--- +title: "Prefix Cache 优化" +sidebar: + order: 50 +--- +## 功能介绍 +xLLM支持prefix_cache匹配。prefix_cache基于mermer_hash，使用lru淘汰策略，提供更极致的匹配效率，同时提高prefix_cache命中率。 +同时对prefix_cache进行了优化，支持continuous_scheduler、chunked_scheduler和zero_evict_scheduler，在prefill之后即更新 +prefix_cache，提高匹配时效性，同时对于chunked_scheduler，支持多阶段chunked_prefill匹配，减少计算量并尽可能减少kv_cache占用。 + +## 使用方式 +prefix_cache已在xLLM实现，并向外暴露gflag参数，控制功能的开关。 + +- 开启zero_evict策略，并设置max_decode_token_per_sequence。 +``` +--enable_prefix_cache=true +``` + +## Cache 感知 DP 路由 + +当使用数据并行（`dp_size > 1`）时，请求可被路由到持有最长 prefix cache 命中的 DP rank，提高跨 rank 的 KV cache 复用率。 + +- 开启 cache 感知 DP 路由： +``` +--enable_prefix_cache=true --enable_prefix_cache_aware_dp_routing=true +``` + +- `prefix_cache_aware_dp_match_threshold`（默认 `0.5`）：选择 cache 亲和 rank 所需的最低 prefix block 命中比例，低于该阈值时回退到空闲 block 均衡。 +- `prefix_cache_aware_dp_imbalance_threshold`（默认 `0.1`）：跨 rank KV 利用率差异上限（`(max_used-min_used)/total_blocks`），超过此值时路由到负载最低的 rank。 + +## 性能效果 +开启prefix_cache之后，在Qwen3-8B模型上，限制TPOT50ms，E2E时延 **下降10%**。 + +:::note +PD分离下的prefix cache支持范围与调度器角色相关，参见[PD分离](/zh/features/disagg_pd/)中的支持配置说明。 +::: diff --git a/docs/src/content/docs/zh/features/xllm_service_overview.md b/docs/src/content/docs/zh/features/xllm_service_overview.md new file mode 100644 index 0000000000..f25b27ccda --- /dev/null +++ b/docs/src/content/docs/zh/features/xllm_service_overview.md @@ -0,0 +1,50 @@ +--- +title: "xLLM Service" +sidebar: + order: 90 +--- +[:simple-github: xLLM Service](https://github.com/xLLM-AI/xllm-service) + +## 简介 + +**xLLM-service** 是一个基于 xLLM 推理引擎开发的服务层框架，为集群化部署提供高效率、高容错、高灵活性的大模型推理服务。 + +xLLM-service 旨在解决企业级服务场景中的关键挑战： + +- 如何于在离线混合部署环境中，保障在线服务的SLA，提升离线任务的资源利用率。 +- 如何适应实际业务中动态变化的请求负载，如输入/输出长度出现剧烈波动。 +- 解决多模态模型请求的性能瓶颈。 +- 保障集群计算实例的高可靠性。 + +#### 背景 +当前，百亿至万亿参数规模的大语言模型正快速部署于智能客服、实时推荐、内容生成等核心业务场景，对国产计算硬件的高效支持已成为低成本推理部署的核心需求。现有推理引擎难以有效适配国产芯片等专用加速器的架构特性，硬件计算单元利用率低、MoE 架构下的负载不均衡与通信开销瓶颈、kv 缓存管理困难等问题，制约了请求的高效推理与系统的可扩展性。xLLM-service + xLLM推理引擎提升了全链路效率，为大语言模型在实际业务中的规模化落地提供了关键技术支撑。 + +--- + +## 整体架构 +xLLM-service 整体架构如图所示: + +![1](../../assets/service_arch.png) + +## 核心组件 + +### ETCD Cluster +用于元信息管理，包括模型，xllm实例，请求等元信息的存储与管理。同时提供xllm节点注册与发现服务。 + +### Fault Tolerance +xLLM-service 提供容错管理，保障服务质量以及稳定性。 + +### Global Scheduler +实现全局感知调度，根据当前系统状态，将请求精准调度至最优实例执行，有效提升整体服务响应效率与资源利用率。 + +### Global KV Cache Manager +负责全局 KV Cache 管理，核心能力包括分布式 KV 缓存感知、Prefix 前缀匹配、KV Cache 动态迁移等，优化缓存资源使用效率。 + +### Instance Manager +聚焦实例全生命周期管理，所有 xllm 实例启动后需向本模块注册，模块基于预设策略，为实例提供调度适配、容错处理等支持。 + +### Event Plane +作为指标与事件中枢，接收各实例上报的 Metrics 数据，对统计指标进行统一收集与整理，为服务调度、容错、扩缩容等决策提供数据支撑。 + +### Planner +承担策略分析与决策职能，基于 Event Plane 上报的 Metrics 数据（含实例运行时指标、机器负载指标等），分析服务扩缩容需求、热点实例扩展必要性，输出资源调整与实例优化策略。 diff --git a/docs/src/content/docs/zh/features/zero_evict_scheduler.md b/docs/src/content/docs/zh/features/zero_evict_scheduler.md new file mode 100644 index 0000000000..a87579fced --- /dev/null +++ b/docs/src/content/docs/zh/features/zero_evict_scheduler.md @@ -0,0 +1,21 @@ +--- +title: "Zero Evict调度器" +sidebar: + order: 22 +--- +## 功能介绍 +xLLM支持zero_evict调度策略。zero_evict调度策略是一种尽可能减少请求淘汰率的调度算法，可以减少淘汰请求的prefill计算，减少TPOT。 +这种调度算法通过模拟轮次，检测请求是否调度可以被调度且不导致其它请求被淘汰。 + +## 使用方式 +上述策略已在xLLM实现，并向外暴露gflag参数，控制功能的开关。 + +- 开启zero_evict策略，并设置max_decode_token_per_sequence。 +``` +--use_zero_evict=true +--max_decode_token_per_sequence=256 +``` + +## 性能效果 +开启zero_evict之后，在Qwen3-8B模型上，限制E2E时延，TPOT时延 **下降27%**。 + diff --git a/docs/src/content/docs/zh/getting_started/launch_xllm.md b/docs/src/content/docs/zh/getting_started/launch_xllm.md new file mode 100644 index 0000000000..2855124710 --- /dev/null +++ b/docs/src/content/docs/zh/getting_started/launch_xllm.md @@ -0,0 +1,243 @@ +--- +title: "启动xllm" +sidebar: + order: 3 +--- +以Qwen3为例，启动xllm的脚本如下，给出的脚本适用于单机单卡和单机多卡，当使用单机多卡时，需要修改`NNODES`（一张卡就代表一个node），以及`ASCEND_RT_VISIBLE_DEVICES`或`CUDA_VISIBLE_DEVICES`或`MLU_VISIBLE_DEVICES`等环境变量。 + +## NPU + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +export ASCEND_RT_VISIBLE_DEVICES=0 +export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 + +MODEL_PATH="/path/to/model/Qwen3-8B" # 模型路径 +MASTER_NODE_ADDR="127.0.0.1:9748" # Master 节点地址（需全局一致） +START_PORT=18000 # 服务起始端口 +START_DEVICE=0 # 起始逻辑设备号 +LOG_DIR="log" # 日志目录 +NNODES=1 # 节点数（当前脚本启动 1 个进程） + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="npu:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --max_memory_utilization=0.86 \ + --block_size=128 \ + --communication_backend="hccl" \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=true \ + --enable_schedule_overlap=true \ + --enable_shm=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +## NVIDIA GPU + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export CUDA_VISIBLE_DEVICES=0 +# for debug +# export CUDA_LAUNCH_BLOCKING=1 + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="cuda:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --block_size=32 \ + --max_memory_utilization=0.8 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_schedule_overlap=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +## MLU + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export MLU_VISIBLE_DEVICES=0 + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="mlu:$DEVICE" \ + --port $PORT \ + --nnodes=$NNODES \ + --master_node_addr=$MASTER_NODE_ADDR \ + --block_size=16 \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +## 海光 DCU + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export HIP_VISIBLE_DEVICES=0 + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="dcu:$DEVICE" \ + --port $PORT \ + --nnodes=$NNODES \ + --master_node_addr=$MASTER_NODE_ADDR \ + --block_size=128 \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +## 沐曦 MACA + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export CUDA_VISIBLE_DEVICES=0 +export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/data/aot/ + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="cuda:$DEVICE" \ + --port $PORT \ + --nnodes=$NNODES \ + --master_node_addr=$MASTER_NODE_ADDR \ + --block_size=128 \ + --max_memory_utilization=0.86 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_schedule_overlap=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +## 摩尔线程 MUSA + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export MUSA_VISIBLE_DEVICES=0 + +MODEL_PATH="/path/to/model/Qwen3.5-27B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="musa:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --block_size=64 \ + --max_memory_utilization=0.8 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=true \ + --enable_schedule_overlap=true \ + --enable_graph=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +详见 [摩尔线程 MUSA](/zh/hardware/musa/)。 diff --git a/docs/src/content/docs/zh/getting_started/multi_machine.md b/docs/src/content/docs/zh/getting_started/multi_machine.md new file mode 100644 index 0000000000..7eed9e114a --- /dev/null +++ b/docs/src/content/docs/zh/getting_started/multi_machine.md @@ -0,0 +1,110 @@ +--- +title: "多机部署" +sidebar: + order: 5 +--- +该示例为两机32卡启动示例，第一台机器服务: +```shell +bash start_deepseek_machine_1.sh +``` +start_deepseek_machine_1.sh 脚本如下: +```bash +#!/bin/bash +set -e + +rm -rf core.* + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 + +MODEL_PATH="/path/to/your/DeepSeek-R1" # 模型路径 +MASTER_NODE_ADDR="123.123.123.123:9748" # Master 节点地址（需全局一致） +LOCAL_HOST=123.123.123.123 # 本机服务启动IP +START_PORT=18000 # 服务起始端口 +START_DEVICE=0 # 起始 NPU 逻辑设备号 +LOG_DIR="log" # 日志目录 +LOCAL_NODES=16 # 单机节点数（当前脚本启动 16 个进程） +NNODES=32 # 总卡数（该示例为2机32卡） + +mkdir -p $LOG_DIR + +for (( i=0; i<$LOCAL_NODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + /path/to/xllm \ + --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --max_memory_utilization=0.86 \ + --max_tokens_per_batch=40000 \ + --max_seqs_per_batch=256 \ + --block_size=128 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --communication_backend="hccl" \ + --enable_schedule_overlap=true \ + --rank_tablefile=./ranktable_2s_32p.json \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +启动第二台机器服务: + +```shell +bash start_deepseek_machine_2.sh +``` +start_deepseek_machine_2.sh 脚本如下: +```bash +#!/bin/bash +set -e + +rm -rf core.* + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 + +MODEL_PATH="/path/to/your/DeepSeek-R1" # 模型路径 +MASTER_NODE_ADDR="123.123.123.123:9748" # Master 节点地址（需全局一致） +LOCAL_HOST=456.456.456.456 # 本机服务启动IP +START_PORT=18000 # 服务起始端口 +START_DEVICE=0 # 起始 NPU 逻辑设备号 +LOG_DIR="log" # 日志目录 +LOCAL_NODES=16 # 单机节点数（当前脚本启动 16 个进程） +NNODES=32 # 总卡数（该示例为2机32卡） + +mkdir -p $LOG_DIR + +for (( i=0; i<$LOCAL_NODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + /path/to/xllm \ + --model $MODEL_PATH \ + --host $LOCAL_HOST \ + --port $PORT \ + --devices="npu:$DEVICE" \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --max_memory_utilization=0.86 \ + --max_tokens_per_batch=40000 \ + --max_seqs_per_batch=256 \ + --block_size=128 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --communication_backend="hccl" \ + --enable_schedule_overlap=true \ + --rank_tablefile=./ranktable_2s_32p.json \ + --node_rank=$((i + LOCAL_NODES)) \ > $LOG_FILE 2>&1 & +done +``` +这里使用了两台机器，可以通过 `--nnodes`设置总卡数，`--node_rank`为全局rank id。 +`--rank_tablefile=./ranktable_2s_32p.json`为构建npu通信域所需文件，可参考[ranktable 生成](https://gitee.com/mindspore/models/blob/master/utils/hccl_tools/README.md)生成。 + diff --git a/docs/src/content/docs/zh/getting_started/offline_service.md b/docs/src/content/docs/zh/getting_started/offline_service.md new file mode 100644 index 0000000000..c9b9fbd9a2 --- /dev/null +++ b/docs/src/content/docs/zh/getting_started/offline_service.md @@ -0,0 +1,44 @@ +--- +title: "离线推理" +sidebar: + order: 7 +--- +为了方便用户快速使用xLLM进行离线推理，我们提供了启动离线推理的python脚本例子 + +## LLM + +LLM推理示例：[:simple-github: https://github.com/xLLM-AI/xllm/blob/main/examples/generate.py](https://github.com/xLLM-AI/xllm/blob/main/examples/generate.py) + +LLM Beam Search 示例：[:simple-github: https://github.com/xLLM-AI/xllm/blob/main/examples/generate_beam_search.py](https://github.com/xLLM-AI/xllm/blob/main/examples/generate_beam_search.py) + +使用 `BeamSearchParams` 设置大于 `1` 的 `beam_width`，然后调用 `llm.beam_search(...)`： + +```python +from xllm import BeamSearchParams, LLM + +llm = LLM(model="/path/models/Qwen2-7B-Instruct", devices="npu:0") +params = BeamSearchParams( + beam_width=2, + top_logprobs=4, + max_tokens=20, +) + +outputs = llm.beam_search( + [{"prompt": "Hello, my name is "}], + params=params, +) +print(outputs[0].sequences[0].text) + +llm.finish() +``` + +LLM Beam Search 使用 `beam_width` 作为开启参数。`top_logprobs` 控制每个解码步用于 beam 扩展的 top-k 候选数量。如果 `top_logprobs` 保持默认值，xLLM 会使用 `beam_width` 作为 top logprob 数量。如果希望每个 beam 考虑更多候选 token，可以将 `top_logprobs` 设置为大于 `beam_width` 的值。这里的 beam-search top-k 不同于采样截断参数 `top_k`。`best_of` 不是 Beam Search 开关，本文档也不使用 `num_return_sequences` 来控制 LLM 返回的 beam 数。 + +## Embedding + +生成Embedding示例：[:simple-github: https://github.com/xLLM-AI/xllm/blob/main/examples/generate_embedding.py](https://github.com/xLLM-AI/xllm/blob/main/examples/generate_embedding.py) + +## VLM + +VLM推理示例：[:simple-github: https://github.com/xLLM-AI/xllm/blob/main/examples/generate_vlm.py](https://github.com/xLLM-AI/xllm/blob/main/examples/generate_vlm.py) + diff --git a/docs/src/content/docs/zh/getting_started/online_service.md b/docs/src/content/docs/zh/getting_started/online_service.md new file mode 100644 index 0000000000..2474f70540 --- /dev/null +++ b/docs/src/content/docs/zh/getting_started/online_service.md @@ -0,0 +1,266 @@ +--- +title: "在线服务" +sidebar: + order: 6 +--- +先按照[xllm启动文档](/zh/getting_started/launch_xllm/)启动xllm服务。下面给出LLM和VLM的客户端调用示例，需要根据实际情况修改其中的参数。 + +## LLM 客户端调用 +### HTTP 调用 + +chat模式： +```bash +curl http://localhost:9977/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen2-7B-Instruct", + "max_tokens": 10, + "temperature": 0, + "stream": true, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "hello xllm" + } + ] + }' +``` + +completions模式： +```bash +curl http://127.0.0.1:9977/v1/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen2-7B-Instruct", + "prompt": "hello xllm", + "max_tokens": 10, + "temperature": 0, + "stream": true + }' +``` + +Beam Search： + +将 `beam_width` 设置为大于 `1` 的值即可开启 LLM Beam Search。`/v1/chat/completions` 和 `/v1/completions` 均支持该参数。beam-search top-k 候选数量在 chat 请求中使用 `top_logprobs` 配置，在 completions 请求中使用数值型 `logprobs` 字段配置。如果这些字段未设置，xLLM 会使用 `beam_width` 作为 top logprob 数量。如果希望每个 beam 考虑更多候选 token，可以将候选数量设置为大于 `beam_width` 的值。这里的 top-k 不同于采样截断参数 `top_k`。`best_of` 不是 Beam Search 开关，本文档也不使用 `num_return_sequences` 来控制 LLM 返回的 beam 数。 + +chat模式： +```bash +curl http://localhost:9977/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen2-7B-Instruct", + "max_tokens": 20, + "temperature": 0, + "stream": false, + "beam_width": 2, + "logprobs": true, + "top_logprobs": 4, + "messages": [ + { + "role": "user", + "content": "请简短介绍 xLLM。" + } + ] + }' +``` + +completions模式： +```bash +curl http://127.0.0.1:9977/v1/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen2-7B-Instruct", + "prompt": "请简短介绍 xLLM。", + "max_tokens": 20, + "temperature": 0, + "stream": false, + "beam_width": 2, + "logprobs": 4 + }' +``` + +sample模式： +```bash +curl http://127.0.0.1:9977/v1/sample \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen2-7B-Instruct", + "prompt": "问题： 是否命中。结论：", + "selector": { + "type": "literal", + "value": "" + }, + "logprobs": 5, + "request_id": "sample-demo-001" + }' +``` + +典型响应： +```json +{ + "id": "sample-demo-001", + "object": "sample_completion", + "created": 1773369600, + "model": "Qwen2-7B-Instruct", + "choices": [ + { + "index": 0, + "text": "True", + "logprobs": { + "tokens": ["True", "False"], + "token_ids": [3456, 7890], + "token_logprobs": [-0.12, -2.31] + }, + "finish_reason": "selector_match" + }, + { + "index": 1, + "text": "", + "logprobs": { + "tokens": [], + "token_ids": [], + "token_logprobs": [] + }, + "finish_reason": "empty_logprobs" + } + ], + "usage": { + "prompt_tokens": 20, + "completion_tokens": 2, + "total_tokens": 22 + } +} +``` + +`/v1/sample` 使用说明： + +- 仅支持 `--backend=llm`，当前不支持 VLM/DiT/Rec。 +- `selector.type` 当前固定为 `literal`，`selector.value` 按 prompt 文本顺序全文匹配。 +- `logprobs` 默认值为 `5`，允许范围为 `[1, 5]`。 +- `choices[i].index` 即该命中的 `sample_id`，与 prompt 中命中顺序一一对应。 +- selector 无命中时返回 `200` 且 `choices=[]`；某命中位点无可用 logprobs 时返回 `finish_reason="empty_logprobs"`。 +- 服务日志只记录 `request_id`、`sample_id`、`match_count`、`model` 等摘要字段，不记录完整 prompt。 + +`/v1/sample` 常见错误语义： + +- 缺少 `model/prompt/selector/selector.value`、`selector.type != literal` 或 `logprobs` 越界时返回 `INVALID_ARGUMENT`。 +- 模型不存在或后端不是 `llm` 时返回 `UNKNOWN`。 +- 并发达到上限时返回 `RESOURCE_EXHAUSTED`。 +- 模型处于 sleep 状态时返回 `UNAVAILABLE`。 + +### Python调用 +```python +import requests +import json + +url = f"http://localhost:9977/v1/chat/completions" +messages = [ + {'role': 'user', 'content': "列出三个国家和他的首都。"} +] + +request_data = { + "model": "Qwen2-7B-Instruct", + "messages": messages, + "stream": False, + "temperature": 0.6, + "max_tokens": 2048, +} + +response = requests.post(url, json=request_data) +if response.status_code != 200: + print(response.status_code, response.text) +else: + ans = json.loads(response.text)["choices"] + print(ans[0]['message']) +``` + +## VLM 客户端调用 +### HTTP API + +```python +import base64 +import requests + +api_url = "http://localhost:12345/v1/chat/completions" +image_url = "" + +def encode_image(url: str) -> str: + with requests.get(url) as response: + response.raise_for_status() + result = base64.b64encode(response.content).decode("utf-8") + + return result + +image_base64 = encode_image(image_url) +payload = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "介绍下这张图片"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"}, + }, + ], + } + ], + "model": "Qwen2.5-VL-7B-Instruct", + "max_completion_tokens": 128, +} + +response = requests.post( + api_url, + json=payload, + headers={"Content-Type": "application/json"} +) +print(response.json()) +``` + +### OpenAI API +```python +from openai import OpenAI +import base64 +import requests + +openai_api_key = "EMPTY" +openai_api_base = "http://localhost:12345/v1" +image_url = "" + +client = OpenAI( + api_key=openai_api_key, + base_url=openai_api_base, +) + +def encode_image(url: str) -> str: + with requests.get(url) as response: + response.raise_for_status() + result = base64.b64encode(response.content).decode("utf-8") + + return result + +image_base64 = encode_image(image_url) +chat_completion = client.chat.completions.create( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "介绍下这张图片"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"}, + }, + ], + } + ], + model="Qwen2.5-VL-7B-Instruct", + max_completion_tokens=128, +) + +result = chat_completion.choices[0].message.content +print("Chat completion output:", result) +``` diff --git a/docs/src/content/docs/zh/getting_started/quick_start.md b/docs/src/content/docs/zh/getting_started/quick_start.md new file mode 100644 index 0000000000..970601afa1 --- /dev/null +++ b/docs/src/content/docs/zh/getting_started/quick_start.md @@ -0,0 +1,200 @@ +--- +title: "快速开始" +sidebar: + order: 1 +--- +## 环境设置 + +所有的镜像都存放在[这里](https://quay.io/repository/jd_xllm/xllm-ai?tab=tags)，下面的docker启动命令以开发镜像为例。 + +### NPU + +下面是我们构建好的开发镜像。 +```bash +# A2 x86 +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-x86-20260306 +# A2 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-arm-20260306 +# A3 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-20260306 +``` + +容器启动命令如下： +```bash +docker run -it \ +--ipc=host \ +-u 0 \ +--name xllm-npu \ +--privileged \ +--network=host \ +--device=/dev/davinci0 \ +--device=/dev/davinci_manager \ +--device=/dev/devmm_svm \ +--device=/dev/hisi_hdc \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ +-v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \ +-v /usr/local/sbin/:/usr/local/sbin/ \ +-v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \ +-v /var/log/npu/slog/:/var/log/npu/slog \ +-v /var/log/npu/profiling/:/var/log/npu/profiling \ +-v /var/log/npu/dump/:/var/log/npu/dump \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +### NVIDIA GPU + +我们提供了NVIDIA GPU使用的[Dockerfile](https://github.com/xLLM-AI/xllm/blob/main/docker/Dockerfile.cuda)，可以构建自定义镜像，当然也可以使用我们根据默认Dockerfile构建的开发镜像： +```bash +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-cuda-x86 +``` + +容器启动命令如下： +```bash +sudo docker run -it \ +--privileged \ +--shm-size '128gb' \ +--ipc=host \ +--net=host \ +--pid=host \ +--name=xllm-cuda \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +### MLU + +我们无法提供MLU镜像，如果您已经拥有了相应的开发镜像，那么可以根据下面的命令启动容器： +```bash +sudo docker run -it \ +--privileged \ +--shm-size '128gb' \ +--ipc=host \ +--net=host \ +--pid=host \ +--name xllm-mlu \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +### 海光 DCU + +下面是我们构建好的开发镜像。 +```bash +docker pull harbor.sourcefind.cn:5443/dcu/admin/base/custom:xllm-dev-dcu-x86-20260617 +``` + +容器启动命令如下： +```bash +docker run -it \ +--ipc=host \ +-u 0 \ +--name xllm-dcu \ +--privileged \ +--network=host \ +--shm-size 256g \ +--device=/dev/kfd \ +--device=/dev/dri \ +--device=/dev/mkfd \ +--security-opt seccomp=unconfined \ +--group-add video \ +-v /opt/hyhal:/opt/hyhal \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +### 沐曦 MACA + +下面是我们构建好的开发镜像。 +```bash +docker pull pub-registry1.metax-tech.com/dev-m01421/xllm-maca3.7.1.9:v1 +``` + +容器启动命令如下： +```bash +docker run -it \ +--ipc=host \ +-u 0 \ +--name xllm-maca \ +--network=host \ +--privileged=true \ +--shm-size 100gb \ +--device=/dev/mxcd \ +--device=/dev/dri \ +--device=/dev/infiniband \ +--security-opt seccomp=unconfined \ +--security-opt apparmor=unconfined \ +--group-add video \ +--ulimit memlock=-1 \ +-v /opt/maca:/opt/maca \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +### 摩尔线程 MUSA + +镜像拉取命令： + +```bash +docker pull registry.mthreads.com/presale/devtech/xllm:0710 +``` + +容器启动命令： + +```bash +docker run -it \ + --ipc=host \ + --network=host \ + --privileged \ + --shm-size=128g \ + --name xllm-musa \ + --device=/dev/mtgpu0 \ + --device=/dev/dri \ + --group-add video \ + --ulimit memlock=-1 \ + -v $HOME:$HOME \ + -w $HOME \ + registry.mthreads.com/presale/devtech/xllm:0710 \ + /bin/bash +``` + +更多细节见 [摩尔线程 MUSA](/zh/hardware/musa/)。 + +## 编译xllm + +如果下载的是release镜像，即tag中带有版本号的镜像，可以跳过此步，因为release镜像自带编译好的xllm二进制文件，可以直接调用`xllm`。 + +下载xllm及依赖 +```bash +git clone https://github.com/xLLM-AI/xllm.git +cd xllm + +# 第一次需要进行pre-commit安装 +pip install pre-commit +pre-commit install + +git submodule update --init --recursive +``` + +在新镜像中，第一次编译xllm耗时较长，因为需要编译vcpkg中的所有依赖，但是后续编译会很快。 +```bash +# 只编译cpp二进制文件 +python setup.py build + +# 编译python wheel +python setup.py bdist_wheel +``` + +## 启动xllm +请参考 [xllm启动方式](/zh/getting_started/launch_xllm/)。 diff --git a/docs/src/content/docs/zh/hardware/ascend_npu.md b/docs/src/content/docs/zh/hardware/ascend_npu.md new file mode 100644 index 0000000000..e87fecac9d --- /dev/null +++ b/docs/src/content/docs/zh/hardware/ascend_npu.md @@ -0,0 +1,100 @@ +--- +title: "昇腾 NPU" +description: "使用 NPU 后端在昇腾硬件上运行 xLLM。" +sidebar: + order: 3 +--- + +在昇腾硬件上部署 xLLM 时使用 NPU 后端。 + +## 镜像和容器启动命令 + +拉取预构建昇腾开发镜像： + +```bash +# A2 x86 +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-x86-20260306 +# A2 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a2-arm-20260306 +# A3 arm +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-a3-arm-20260306 +``` + +启动容器： + +```bash +docker run -it \ +--ipc=host \ +-u 0 \ +--name xllm-npu \ +--privileged \ +--network=host \ +--device=/dev/davinci0 \ +--device=/dev/davinci_manager \ +--device=/dev/devmm_svm \ +--device=/dev/hisi_hdc \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ +-v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \ +-v /usr/local/sbin/:/usr/local/sbin/ \ +-v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \ +-v /var/log/npu/slog/:/var/log/npu/slog \ +-v /var/log/npu/profiling/:/var/log/npu/profiling \ +-v /var/log/npu/dump/:/var/log/npu/dump \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +## 服务启动命令 + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +export ASCEND_RT_VISIBLE_DEVICES=0 +export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 + +MODEL_PATH="/path/to/model/Qwen3-8B" # 模型路径 +MASTER_NODE_ADDR="127.0.0.1:9748" # Master 节点地址（需全局一致） +START_PORT=18000 # 服务起始端口 +START_DEVICE=0 # 起始逻辑设备号 +LOG_DIR="log" # 日志目录 +NNODES=1 # 节点数（当前脚本启动 1 个进程） + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="npu:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --max_memory_utilization=0.86 \ + --block_size=128 \ + --communication_backend="hccl" \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=true \ + --enable_schedule_overlap=true \ + --enable_shm=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +分布式 NPU 服务通常需要设置 `--communication_backend="hccl"`，并保证所有 worker 的 `--master_node_addr`、`--node_rank`、`--nnodes` 一致；需要 rank table 的部署还应提前准备对应配置。 + +## 注意事项 + +- A2 和 A3 开发镜像维护在 [快速开始](/zh/getting_started/quick_start/) 中。 +- NPU 启动示例通常会启用 HCCL 通信，并按业务负载调整 `--block_size`、`--max_memory_utilization`、chunked prefill 和 schedule overlap 等参数。 +- 昇腾 kernel 适配和开发细节见 TileLang kernel 开发指南。 diff --git a/docs/src/content/docs/zh/hardware/cambricon_mlu.md b/docs/src/content/docs/zh/hardware/cambricon_mlu.md new file mode 100644 index 0000000000..a1f953e388 --- /dev/null +++ b/docs/src/content/docs/zh/hardware/cambricon_mlu.md @@ -0,0 +1,69 @@ +--- +title: "寒武纪 MLU" +description: "使用 MLU 后端在寒武纪设备上运行 xLLM。" +sidebar: + order: 4 +--- + +在寒武纪设备上部署 xLLM 时使用 MLU 后端。 + +## 镜像和容器启动命令 + +当前文档不提供公开 MLU 镜像。如果您已经拥有了相应的开发镜像，可以根据下面的命令启动容器： + +```bash +sudo docker run -it \ +--privileged \ +--shm-size '128gb' \ +--ipc=host \ +--net=host \ +--pid=host \ +--name xllm-mlu \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +## 服务启动命令 + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export MLU_VISIBLE_DEVICES=0 + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="mlu:$DEVICE" \ + --port $PORT \ + --nnodes=$NNODES \ + --master_node_addr=$MASTER_NODE_ADDR \ + --block_size=16 \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +单卡部署时 `` 通常从 `0` 开始。更大规模部署中，需要让设备编号、`--node_rank`、`--nnodes` 和每个 worker 的端口保持一致。 + +## 注意事项 + +- 当前文档不提供公开 MLU 镜像，需要使用已有 MLU 开发镜像，并配合上面的容器启动命令。 +- 当前 MLU 启动示例使用 `--block_size=16`。 +- 选择 MLU 部署目标前，先在 [模型支持列表](/zh/supported_models/) 中确认模型覆盖情况。 diff --git a/docs/src/content/docs/zh/hardware/dcu.md b/docs/src/content/docs/zh/hardware/dcu.md new file mode 100644 index 0000000000..5397b7f9a9 --- /dev/null +++ b/docs/src/content/docs/zh/hardware/dcu.md @@ -0,0 +1,81 @@ +--- +title: "海光 DCU" +description: "使用海光 DCU 后端在海光 DCU 硬件上运行 xLLM。" +sidebar: + order: 5 +--- + +在海光 DCU 硬件上部署 xLLM 时使用海光 DCU 后端。 + +## 镜像和容器启动命令 + +拉取海光 DCU 开发镜像： + +```bash +docker pull harbor.sourcefind.cn:5443/dcu/admin/base/custom:xllm-dev-dcu-x86-20260617 +``` + +启动容器： + +```bash +docker run -it \ +--ipc=host \ +-u 0 \ +--name xllm-dcu \ +--privileged \ +--network=host \ +--shm-size 256g \ +--device=/dev/kfd \ +--device=/dev/dri \ +--device=/dev/mkfd \ +--security-opt seccomp=unconfined \ +--group-add video \ +-v /opt/hyhal:/opt/hyhal \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +## 服务启动命令 + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export HIP_VISIBLE_DEVICES=0 + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="dcu:$DEVICE" \ + --port $PORT \ + --nnodes=$NNODES \ + --master_node_addr=$MASTER_NODE_ADDR \ + --block_size=128 \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +单卡部署时 `` 通常从 `0` 开始。多 worker 部署中，需要让设备编号、`--node_rank`、`--nnodes` 和服务端口保持一致。 + +## 注意事项 + +- 当前文档在 [快速开始](/zh/getting_started/quick_start/) 中列出了海光 DCU 开发镜像。 +- 海光 DCU 容器启动需要挂载 `/dev/kfd`、`/dev/dri`、`/dev/mkfd` 等设备；上面的命令已包含这些挂载。 +- 选择海光 DCU 部署目标前，先在 [模型支持列表](/zh/supported_models/) 中确认模型覆盖情况。 diff --git a/docs/src/content/docs/zh/hardware/metax_maca.md b/docs/src/content/docs/zh/hardware/metax_maca.md new file mode 100644 index 0000000000..4f9890a8bb --- /dev/null +++ b/docs/src/content/docs/zh/hardware/metax_maca.md @@ -0,0 +1,88 @@ +--- +title: "沐曦 MACA" +description: "使用沐曦 MACA 后端在沐曦 MACA 硬件上运行 xLLM。" +sidebar: + order: 6 +--- + +在沐曦 MACA 硬件上部署 xLLM 时使用沐曦 MACA 后端。 + +## 镜像和容器启动命令 + +拉取沐曦 MACA 开发镜像： + +```bash +docker pull pub-registry1.metax-tech.com/dev-m01421/xllm-maca3.7.1.9:v1 +``` + +启动容器： + +```bash +docker run -it \ +--ipc=host \ +-u 0 \ +--name xllm-maca \ +--network=host \ +--privileged=true \ +--shm-size 100gb \ +--device=/dev/mxcd \ +--device=/dev/dri \ +--device=/dev/infiniband \ +--security-opt seccomp=unconfined \ +--security-opt apparmor=unconfined \ +--group-add video \ +--ulimit memlock=-1 \ +-v /opt/maca:/opt/maca \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +## 服务启动命令 + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export CUDA_VISIBLE_DEVICES=0 +export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/data/aot/ + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="cuda:$DEVICE" \ + --port $PORT \ + --nnodes=$NNODES \ + --master_node_addr=$MASTER_NODE_ADDR \ + --block_size=128 \ + --max_memory_utilization=0.86 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_schedule_overlap=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +单卡部署时 `` 通常从 `0` 开始。多 worker 部署中，需要让设备编号、`--node_rank`、`--nnodes` 和服务端口保持一致。 + +## 注意事项 + +- 当前文档在 [快速开始](/zh/getting_started/quick_start/) 中列出了沐曦 MACA 开发镜像。 +- 沐曦MACA 容器启动需要挂载 `/dev/mxcd`、`/dev/dri`、`/dev/infiniband` 等设备；上面的命令已包含这些挂载。 +- 在MetaX MACA容器中编译XLLM命令: python setup.py build --device maca diff --git a/docs/src/content/docs/zh/hardware/musa.md b/docs/src/content/docs/zh/hardware/musa.md new file mode 100644 index 0000000000..2a2fae5242 --- /dev/null +++ b/docs/src/content/docs/zh/hardware/musa.md @@ -0,0 +1,84 @@ +--- +title: "摩尔线程 MUSA" +description: "在摩尔线程 MUSA GPU 上使用 xLLM 进行大模型推理部署。" +sidebar: + order: 7 +--- + +在摩尔线程（Mthreads）MUSA GPU 上部署 xLLM 时使用 MUSA 设备后端（`--devices=musa:`）。 + +## 镜像和容器启动命令 + +拉取摩尔线程 xLLM 镜像： + +```bash +docker pull registry.mthreads.com/presale/devtech/xllm:0710 +``` + +启动容器： + +```bash +docker run -it \ + --ipc=host \ + --network=host \ + --privileged \ + --shm-size=128g \ + --name xllm-musa \ + --device=/dev/mtgpu0 \ + --device=/dev/dri \ + --group-add video \ + --ulimit memlock=-1 \ + -v $HOME:$HOME \ + -w $HOME \ + registry.mthreads.com/presale/devtech/xllm:0710 \ + /bin/bash +``` + +按实际 GPU 数量增加 `--device=/dev/mtgpuN`；选择物理卡使用 `export MUSA_VISIBLE_DEVICES=0`。 + +## 服务启动命令 + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export MUSA_VISIBLE_DEVICES=0 + +MODEL_PATH="/path/to/model/Qwen3.5-27B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="musa:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --block_size=64 \ + --max_memory_utilization=0.8 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=true \ + --enable_schedule_overlap=true \ + --enable_graph=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +单卡部署时逻辑设备号通常从 `0` 开始。多卡或多机场景中，需要让设备编号、`--node_rank`、`--nnodes` 和服务端口与部署拓扑保持一致。 + +## 注意事项 + +- 镜像与容器启动说明亦见 [快速开始](/zh/getting_started/quick_start/)。 +- 启动参数与多机部署见 [启动 xllm](/zh/getting_started/launch_xllm/)。 diff --git a/docs/src/content/docs/zh/hardware/nvidia_gpu.md b/docs/src/content/docs/zh/hardware/nvidia_gpu.md new file mode 100644 index 0000000000..120510fd72 --- /dev/null +++ b/docs/src/content/docs/zh/hardware/nvidia_gpu.md @@ -0,0 +1,81 @@ +--- +title: "NVIDIA GPU" +description: "使用 CUDA 后端在 NVIDIA GPU 上运行 xLLM。" +sidebar: + order: 2 +--- + +在 NVIDIA GPU 上部署 xLLM 时使用 CUDA 后端。 + +## 镜像和容器启动命令 + +拉取 CUDA 开发镜像： + +```bash +docker pull quay.io/jd_xllm/xllm-ai:xllm-dev-cuda-x86 +``` + +启动容器： + +```bash +sudo docker run -it \ +--privileged \ +--shm-size '128gb' \ +--ipc=host \ +--net=host \ +--pid=host \ +--name=xllm-cuda \ +-v $HOME:$HOME \ +-w $HOME \ + \ +/bin/bash +``` + +## 服务启动命令 + +```bash +#!/bin/bash +set -e + +rm -rf core.* + +export CUDA_VISIBLE_DEVICES=0 +# for debug +# export CUDA_LAUNCH_BLOCKING=1 + +MODEL_PATH="/path/to/model/Qwen3-8B" +MASTER_NODE_ADDR="127.0.0.1:9748" +START_PORT=18000 +START_DEVICE=0 +LOG_DIR="log" +NNODES=1 + +mkdir -p $LOG_DIR + +for (( i=0; i<$NNODES; i++ )) +do + PORT=$((START_PORT + i)) + DEVICE=$((START_DEVICE + i)) + LOG_FILE="$LOG_DIR/node_$i.log" + xllm \ + --model $MODEL_PATH \ + --devices="cuda:$DEVICE" \ + --port $PORT \ + --master_node_addr=$MASTER_NODE_ADDR \ + --nnodes=$NNODES \ + --block_size=32 \ + --max_memory_utilization=0.8 \ + --enable_prefix_cache=false \ + --enable_chunked_prefill=false \ + --enable_schedule_overlap=true \ + --node_rank=$i \ > $LOG_FILE 2>&1 & +done +``` + +单卡部署时 `` 通常从 `0` 开始。多卡或多机场景中，需要让设备编号、`--node_rank`、`--nnodes` 和服务端口与部署拓扑保持一致。 + +## 注意事项 + +- CUDA 开发镜像和 Dockerfile 入口维护在 [快速开始](/zh/getting_started/quick_start/) 中。 +- 当前 CUDA 启动示例使用 `--block_size=32` 和 `--max_memory_utilization=0.8`；实际部署时按模型规模和 GPU 显存调整。 +- CUDA timeline 采集入口见 [在线性能采集](/zh/dev_guide/online_profiling/)。 diff --git a/docs/src/content/docs/zh/hardware/overview.md b/docs/src/content/docs/zh/hardware/overview.md new file mode 100644 index 0000000000..39b5990dcf --- /dev/null +++ b/docs/src/content/docs/zh/hardware/overview.md @@ -0,0 +1,24 @@ +--- +title: "硬件平台" +description: "xLLM 支持硬件平台的环境、启动和模型支持入口。" +sidebar: + order: 1 +--- + +xLLM 支持多种加速器后端，用于大模型推理部署。本章节汇总不同硬件平台的环境准备、运行时设备选择、服务启动和模型支持入口。 + +## 平台指南 + +- [NVIDIA GPU](/zh/hardware/nvidia_gpu/) - CUDA 后端环境和启动入口。 +- [昇腾 NPU](/zh/hardware/ascend_npu/) - 昇腾 NPU 环境、运行时变量和 HCCL 启动注意事项。 +- [寒武纪 MLU](/zh/hardware/cambricon_mlu/) - MLU 后端环境和启动入口。 +- [海光 DCU](/zh/hardware/dcu/) - 海光 DCU 后端环境和启动入口。 +- [沐曦 MACA](/zh/hardware/metax_maca/) - 沐曦 MACA 后端环境和启动入口。 +- [摩尔线程 MUSA](/zh/hardware/musa/) - 摩尔线程 MUSA GPU 镜像启动入口。 + +## 通用流程 + +1. 根据各平台指南中的显式命令准备对应平台的容器镜像。 +2. 在容器内编译 xLLM，或直接使用已经包含 `xllm` 的 release 镜像。 +3. 按 [启动 xllm](/zh/getting_started/launch_xllm/) 中对应平台的设备后端启动服务。 +4. 在 [模型支持列表](/zh/supported_models/) 中确认模型和模态覆盖情况。 diff --git a/docs/src/content/docs/zh/index.md b/docs/src/content/docs/zh/index.md new file mode 100644 index 0000000000..9b410093e7 --- /dev/null +++ b/docs/src/content/docs/zh/index.md @@ -0,0 +1,61 @@ +--- +title: "xLLM" +description: "xLLM 文档首页" +head: + - tag: title + content: "xLLM Documentation" +--- +![xLLM](../assets/logo_with_llm.png) + +## 简介 + +**xLLM** 是一个高效且易用的开源智能推理框架，为模型在国产芯片上的推理提供企业级服务保障与高性能引擎计算能力。 + +#### 背景 +当前，百亿至万亿参数规模的大语言模型正快速部署于智能客服、实时推荐、内容生成等核心业务场景，对国产计算硬件的高效支持已成为低成本推理部署的核心需求。现有推理引擎难以有效适配国产芯片等专用加速器的架构特性，硬件计算单元利用率低、MoE 架构下的负载不均衡与通信开销瓶颈、kv 缓存管理困难等问题，制约了请求的高效推理与系统的可扩展性。xLLM 推理引擎提升了 “通信 - 计算 - 存储” 全链路的资源利用效率，为大语言模型在实际业务中的规模化落地提供了关键技术支撑。 + +--- + +## 核心特性 +xLLM 提供了强大的智能计算能力，通过硬件系统的算力优化与算法驱动的决策控制，联合加速推理过程，实现高吞吐、低延迟的分布式推理服务。 + +### 全图化/多层流水线执行编排 + +- 框架调度层的异步解耦调度，减少计算空泡； +- 模型图层的计算和通信异步并行，重叠计算与通信； +- 算子内核层的异构计算单元深度流水，重叠计算与访存。 + +### 动态shape的图执行优化 + +- 基于参数化与多图缓存方法的动态尺寸适配，提升静态图灵活性； +- 受管控的显存池，保证地址安全可复用； +- 集成适配性能关键的自定义算子（如 *PageAttention*, *AllReduce*）。 + +### MoE算子优化 + +- *GroupMatmul* 优化，提升计算效率； +- *Chunked Prefill* 优化，支撑长序列输入。 + +### 高效显存优化 + +- 离散物理内存与连续虚拟内存的映射管理； +- 按需分配内存空间，减少内存碎片与浪费； +- 智能调度内存空间，增加内存页复用，减小分配延迟； +- 国产芯片相应算子适配。 + +### 全局多级KV Cache管理 + +- 多级缓存的kv智能卸载与预取； +- 以kv cache为中心的分布式存储架构； +- 多节点间kv的智能传输路由。 + +### 算法优化 + +- 投机推理优化，多核并行提升效率； +- MoE专家的动态负载均衡，实现专家分布的高效调整。 + +## 设计文档 + +- [Graph Mode 设计文档](/zh/design/graph_mode_design/) +- [生成式推荐设计文档](/zh/design/generative_recommendation_design/) + diff --git a/docs/src/content/docs/zh/supported_models.md b/docs/src/content/docs/zh/supported_models.md new file mode 100644 index 0000000000..5cef0b4365 --- /dev/null +++ b/docs/src/content/docs/zh/supported_models.md @@ -0,0 +1,49 @@ +--- +title: "模型支持列表" +sidebar: + order: 20 +--- +## LLM +| | NPU | MLU | ILU | 海光 DCU | +| ------------------------ | :---: | :---: | :---: | :---: | +| DeepSeek-V3/R1/V3.1 | ✅ | ✅ | ❌ | ❌ | +| DeepSeek-V3.2 | ✅ | ✅ | ❌ | ❌ | +| DeepSeek-R1-Distill-Qwen | ✅ | ❌ | ❌ | ❌ | +| Qwen2/2.5/QwQ | ✅ | ✅ | ✅ | ✅ | +| Qwen3 | ✅ | ✅ | ✅ | ✅ | +| Qwen3 Moe | ✅ | ✅ | ✅ | ✅ | +| Kimi-k2 | ✅ | ❌ | ❌ | ❌ | +| Llama2/3 | ✅ | ❌ | ✅ | ❌ | +| GLM4.5 | ✅ | ❌ | ❌ | ❌ | +| GLM4.6 | ✅ | ❌ | ❌ | ❌ | +| GLM-4.7 | ✅ | ❌ | ❌ | ❌ | +| GLM-5 | ✅ | ❌ | ❌ | ❌ | + +## VLM +| | NPU | MLU | ILU | 海光 DCU | +| ------------ | :---: | :---: | :---: | :---: | +| MiniCPM-V | ✅ | ❌ | ❌ | ❌ | +| MiMo-VL | ✅ | ❌ | ❌ | ❌ | +| Qwen2.5-VL | ✅ | ✅ | ❌ | ✅ | +| Qwen3-VL | ✅ | ✅ | ❌ | ✅ | +| Qwen3-VL-MoE | ✅ | ✅ | ❌ | ✅ | +| GLM-4.6V | ✅ | ❌ | ❌ | ❌ | +| VLM-R1 | ✅ | ❌ | ❌ | ❌ | + +## Rerank +| | NPU | MLU | ILU | 海光 DCU | +| -------------- | :---: | :---: | :---: | :---: | +| Qwen3-Reranker | ✅ | ❌ | ❌ | ❌ | + +## DiT +| | NPU | MLU | ILU | 海光 DCU | +| ---- | :---: | :---: | :---: | :---: | +| Flux | ✅ | ❌ | ❌ | ❌ | + +## Rec +| | NPU | MLU | ILU | 海光 DCU | +| --- | :---: | :---: | :---: | :---: | +| OneRec | ✅ | ❌ | ❌ | ❌ | +| Qwen2 | ✅ | ❌ | ❌ | ❌ | +| Qwen2.5 | ✅ | ❌ | ❌ | ❌ | +| Qwen3 | ✅ | ❌ | ❌ | ❌ | diff --git a/docs/src/env.d.ts b/docs/src/env.d.ts new file mode 100644 index 0000000000..c5d3ca4dd5 --- /dev/null +++ b/docs/src/env.d.ts @@ -0,0 +1,21 @@ +/// + +declare module 'virtual:starlight/components/Search' { + const Search: typeof import('@astrojs/starlight/components/Search.astro').default; + export default Search; +} + +declare module 'virtual:starlight/components/SiteTitle' { + const SiteTitle: typeof import('@astrojs/starlight/components/SiteTitle.astro').default; + export default SiteTitle; +} + +declare module 'virtual:starlight/components/SocialIcons' { + const SocialIcons: typeof import('@astrojs/starlight/components/SocialIcons.astro').default; + export default SocialIcons; +} + +declare module 'virtual:starlight/components/ThemeSelect' { + const ThemeSelect: typeof import('@astrojs/starlight/components/ThemeSelect.astro').default; + export default ThemeSelect; +} diff --git a/docs/src/pages/index.astro b/docs/src/pages/index.astro new file mode 100644 index 0000000000..96453371cd --- /dev/null +++ b/docs/src/pages/index.astro @@ -0,0 +1,23 @@ +--- +import Favicons from '../components/Favicons.astro'; + +const defaultLocalePath = '/en/'; +--- + + + + + + + + + + Redirecting to English docs + + + + Redirecting to English docs + + diff --git a/docs/src/plugins/rehypeSimpleIcons.mjs b/docs/src/plugins/rehypeSimpleIcons.mjs new file mode 100644 index 0000000000..c15e4ae61f --- /dev/null +++ b/docs/src/plugins/rehypeSimpleIcons.mjs @@ -0,0 +1,86 @@ +const simpleIconDefinitions = { + ':simple-github:': { + className: 'simple-icon-github', + path: 'M12 .3a12 12 0 0 0-3.8 23.38c.6.12.83-.26.83-.57L9 21.07c-3.34.72-4.04-1.61-4.04-1.61-.55-1.39-1.34-1.76-1.34-1.76-1.08-.74.09-.73.09-.73 1.2.09 1.83 1.24 1.83 1.24 1.08 1.83 2.81 1.3 3.5 1 .1-.78.42-1.31.76-1.61-2.67-.3-5.47-1.33-5.47-5.93 0-1.31.47-2.38 1.24-3.22-.14-.3-.54-1.52.1-3.18 0 0 1-.32 3.3 1.23a11.5 11.5 0 0 1 6 0c2.28-1.55 3.29-1.23 3.29-1.23.64 1.66.24 2.88.12 3.18a4.65 4.65 0 0 1 1.23 3.22c0 4.61-2.8 5.63-5.48 5.92.42.36.81 1.1.81 2.22l-.01 3.29c0 .31.2.69.82.57A12 12 0 0 0 12 .3Z', + }, +}; + +const simpleIconPattern = new RegExp( + `(${Object.keys(simpleIconDefinitions) + .map((name) => name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')) + .join('|')})`, + 'g' +); +const ignoredTags = new Set(['code', 'pre', 'script', 'style']); + +export default function rehypeSimpleIcons() { + return function transform(tree) { + replaceSimpleIconText(tree); + }; +} + +function replaceSimpleIconText(parent) { + if (!Array.isArray(parent.children)) return; + if (parent.type === 'element' && ignoredTags.has(parent.tagName)) return; + + for (let index = 0; index < parent.children.length; index += 1) { + const child = parent.children[index]; + if (child.type === 'text' && simpleIconPattern.test(child.value)) { + simpleIconPattern.lastIndex = 0; + const replacementNodes = createReplacementNodes(child.value); + parent.children.splice(index, 1, ...replacementNodes); + index += replacementNodes.length - 1; + continue; + } + + simpleIconPattern.lastIndex = 0; + replaceSimpleIconText(child); + } +} + +function createReplacementNodes(value) { + const nodes = []; + let lastIndex = 0; + let match; + + simpleIconPattern.lastIndex = 0; + while ((match = simpleIconPattern.exec(value)) !== null) { + if (match.index > lastIndex) { + nodes.push({ type: 'text', value: value.slice(lastIndex, match.index) }); + } + + nodes.push(createSimpleIconNode(match[0])); + lastIndex = match.index + match[0].length; + } + + if (lastIndex < value.length) { + nodes.push({ type: 'text', value: value.slice(lastIndex) }); + } + + return nodes; +} + +function createSimpleIconNode(name) { + const icon = simpleIconDefinitions[name]; + return { + type: 'element', + tagName: 'svg', + properties: { + ariaHidden: 'true', + className: ['simple-icon', icon.className], + fill: 'currentColor', + focusable: 'false', + height: '1em', + viewBox: '0 0 24 24', + width: '1em', + }, + children: [ + { + type: 'element', + tagName: 'path', + properties: { d: icon.path }, + children: [], + }, + ], + }; +} diff --git a/docs/src/styles/theme.css b/docs/src/styles/theme.css new file mode 100644 index 0000000000..2676f3aeca --- /dev/null +++ b/docs/src/styles/theme.css @@ -0,0 +1,229 @@ +:root, +::backdrop, +:root[data-theme='light'], +[data-theme='light'] ::backdrop { + --docs-red: #EB2700; + --docs-red-muted: color-mix(in srgb, var(--docs-red) 80%, transparent); + --docs-black: #080808; + --sl-color-accent-low: color-mix(in srgb, var(--docs-red) 18%, var(--sl-color-black)); + --sl-color-accent: var(--docs-red); + --sl-color-accent-high: var(--docs-red); + --sl-color-bg-accent: var(--docs-red); + --sl-color-text-accent: var(--docs-red); + --sl-content-width: 60rem; +} + +:root:not([data-theme='light']), +:root:not([data-theme='light']) ::backdrop { + --sl-color-white: #f7f7f7; + --sl-color-gray-1: #e8e8e8; + --sl-color-gray-2: #d0d0d0; + --sl-color-gray-3: #a8a8a8; + --sl-color-gray-4: #747474; + --sl-color-gray-5: #343434; + --sl-color-gray-6: #1d1d1d; + --sl-color-gray-7: #111111; + --sl-color-black: var(--docs-black); + --sl-color-bg: var(--docs-black); + --sl-color-bg-nav: var(--docs-black); + --sl-color-bg-sidebar: var(--docs-black); + --sl-color-bg-inline-code: var(--sl-color-gray-6); + --sl-color-text: var(--sl-color-gray-2); + --sl-color-text-invert: var(--docs-black); + --sl-color-hairline-light: var(--sl-color-gray-5); + --sl-color-hairline: var(--sl-color-gray-6); + --sl-color-hairline-shade: var(--docs-black); + --sl-color-backdrop-overlay: color-mix(in srgb, var(--docs-black) 72%, transparent); + --sl-rapide-ui-bg-color: var(--docs-black); + --sl-rapide-ui-header-bg-color: color-mix(in srgb, var(--docs-black) 88%, transparent); + --sl-rapide-ui-border-color: color-mix(in srgb, var(--docs-red) 20%, var(--sl-color-gray-6)); +} + +.sidebar-pane a > span:first-child, +.sidebar-pane summary .group-label { + display: inline-block; + transform: scale(1); + transform-origin: left center; + transition: + color 160ms ease, + transform 220ms cubic-bezier(0.34, 1.56, 0.64, 1); +} + +.sidebar-pane summary .group-label .large { + transition: color 160ms ease; +} + +.sidebar-pane a:is(:hover, :focus-visible), +.sidebar-pane summary:is(:hover, :focus-visible), +.sidebar-pane a:is(:hover, :focus-visible) > span:first-child, +.sidebar-pane summary:is(:hover, :focus-visible) .group-label, +.sidebar-pane summary:is(:hover, :focus-visible) .group-label .large { + color: var(--docs-red) !important; +} + +.sidebar-pane a:is(:hover, :focus-visible) > span:first-child, +.sidebar-pane summary:is(:hover, :focus-visible) .group-label { + transform: scale(1.035); +} + +@media (prefers-reduced-motion: reduce) { + .sidebar-pane a > span:first-child, + .sidebar-pane summary .group-label, + .sidebar-pane summary .group-label .large { + transition: color 160ms ease; + } + + .sidebar-pane a:is(:hover, :focus-visible) > span:first-child, + .sidebar-pane summary:is(:hover, :focus-visible) .group-label { + transform: none; + } +} + +starlight-toc a > span { + display: inline-block; + transform: scale(1); + transform-origin: left center; + transition: + color 160ms ease, + transform 220ms cubic-bezier(0.34, 1.56, 0.64, 1); +} + +starlight-toc a:is(:hover, :focus-visible) { + color: var(--docs-red-muted); +} + +starlight-toc a[aria-current='true'], +starlight-toc a[aria-current='true']:is(:hover, :focus-visible) { + color: var(--docs-red) !important; +} + +.sl-markdown-content .simple-icon { + display: inline-block; + flex: 0 0 auto; + height: 1em; + margin-inline-end: 0.25em; + vertical-align: -0.125em; + width: 1em; +} + +starlight-toc a:is(:hover, :focus-visible) > span { + transform: scale(1.035); +} + +header.header .language-menu { + position: relative; +} + +header.header .language-menu summary { + align-items: center; + background-clip: padding-box; + border: 1px solid transparent; + border-radius: 0.625rem; + color: var(--sl-color-gray-2); + cursor: pointer; + display: flex; + font-size: var(--sl-text-sm); + font-weight: 500; + gap: 0.35rem; + line-height: 1; + list-style: none; + padding: 0.55rem 0.75rem; + transform: scale(1); + transform-origin: center; + transition: + background-color 160ms ease, + border-color 160ms ease, + color 160ms ease, + transform 260ms cubic-bezier(0.34, 1.56, 0.64, 1); + user-select: none; + white-space: nowrap; + will-change: transform; +} + +header.header .language-menu summary::-webkit-details-marker { + display: none; +} + +header.header .language-menu summary:is(:hover, :focus-visible), +header.header .language-menu[open] summary { + background-color: color-mix(in srgb, var(--sl-color-accent) 8%, transparent); + border-color: color-mix(in srgb, var(--sl-color-accent) 26%, transparent); + color: var(--sl-color-text-accent); + transform: scale(1.045); +} + +header.header .language-menu svg { + fill: none; + flex: 0 0 auto; + height: 1rem; + stroke: currentColor; + stroke-linecap: round; + stroke-linejoin: round; + stroke-width: 2; + width: 1rem; +} + +header.header .language-menu .chevron-icon { + transition: transform 160ms ease; +} + +header.header .language-menu[open] .chevron-icon { + transform: rotate(180deg); +} + +header.header .language-options { + background-color: color-mix(in srgb, var(--sl-color-bg-nav) 96%, transparent); + border: 1px solid var(--sl-color-gray-5); + border-radius: 0.625rem; + box-shadow: 0 0.85rem 2rem rgba(0, 0, 0, 0.12); + display: grid; + gap: 0.125rem; + inset-block-start: calc(100% + 0.4rem); + inset-inline-end: 0; + min-width: 100%; + padding: 0.25rem; + position: absolute; + z-index: calc(var(--sl-z-index-navbar) + 1); +} + +header.header .language-options a { + border-radius: 0.5rem; + color: var(--sl-color-gray-2); + font-size: var(--sl-text-sm); + font-weight: 500; + line-height: 1; + padding: 0.55rem 0.75rem; + text-align: start; + text-decoration: none; + white-space: nowrap; +} + +header.header .language-options a:is(:hover, :focus-visible), +header.header .language-options a[aria-current='true'] { + background-color: color-mix(in srgb, var(--sl-color-accent) 10%, transparent); + color: var(--sl-color-text-accent); + outline: none; +} + +@media (prefers-reduced-motion: reduce) { + starlight-toc a > span { + transition: color 160ms ease; + } + + starlight-toc a:is(:hover, :focus-visible) > span, + header.header .language-menu summary:is(:hover, :focus-visible), + header.header .language-menu[open] summary { + transform: none; + } + + header.header .language-menu summary { + transition: + background-color 160ms ease, + border-color 160ms ease, + color 160ms ease; + } + + header.header .language-menu .chevron-icon { + transition: none; + } +} diff --git a/docs/tsconfig.json b/docs/tsconfig.json new file mode 100644 index 0000000000..8bf91d3bb9 --- /dev/null +++ b/docs/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "astro/tsconfigs/strict", + "include": [".astro/types.d.ts", "**/*"], + "exclude": ["dist"] +}