diff --git a/.gitignore b/.gitignore index bbb89ac55..e5e4be9b5 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,6 @@ changelog/* .atmos .python-version yarn.lock + +**/llms.txt +**/llms-full.txt diff --git a/docusaurus.config.js b/docusaurus.config.js index 3c4de4a23..44d90be9a 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -100,6 +100,30 @@ async function createConfig() { redirects: getStaticRedirects(), }, ], + [ + 'docusaurus-plugin-llms', + { + generateLLMsTxt: true, + generateLLMsFullTxt: true, + docsDir: 'docs', + includeBlog: true, + includeOrder: [ + 'into/*', + 'quickstart/*', + 'jumpstart/*', + 'learn/*', + 'layers/*', + 'best-practices/*', + 'reference/*', + 'components/*', + 'modules/*', + 'github-actions/*', + 'resources/*', + 'community/*', + 'support/*', + ], + }, + ], redirectsPlugin, async function AddTailwindCSS(context, options) { return { @@ -130,7 +154,7 @@ async function createConfig() { showLastUpdateAuthor: true, // Ignore tags missing from tags.yml // There are many from components and GHA, it's too much noise without being helpful - onInlineTags: 'ignore', + onInlineTags: 'ignore', tags: 'tags.yml', include: ['**/*.md', '**/*.mdx'] }, diff --git a/package-lock.json b/package-lock.json index 1aa665eec..109140f51 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,6 +39,7 @@ "clsx": "^2.1.1", "custom-loaders": "file:plugins/custom-loaders", "docusaurus-plugin-image-zoom": "^2.0.0", + "docusaurus-plugin-llms": "^0.2.2", "docusaurus-plugin-sass": "^0.2.5", "docusaurus-plugin-sentry": "^2.0.0", "html-loader": "^5.0.0", @@ -7512,6 +7513,59 @@ "@docusaurus/theme-classic": ">=3.0.0" } }, + "node_modules/docusaurus-plugin-llms": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/docusaurus-plugin-llms/-/docusaurus-plugin-llms-0.2.2.tgz", + "integrity": "sha512-DZlZ6cv9p5poFE00Qg78aurBNWhLa4o0VhH4kI33DUT0y4ydlFEJJbf8Bks9BuuGPFbY/Guebn+hRc2QymMImg==", + "license": "MIT", + "dependencies": { + "gray-matter": "^4.0.3", + "minimatch": "^9.0.3", + "yaml": "^2.8.1" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "@docusaurus/core": "^3.0.0" + } + }, + "node_modules/docusaurus-plugin-llms/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/docusaurus-plugin-llms/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/docusaurus-plugin-llms/node_modules/yaml": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.1.tgz", + "integrity": "sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + } + }, "node_modules/docusaurus-plugin-sass": { "version": "0.2.5", "resolved": "https://registry.npmjs.org/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.5.tgz", diff --git a/package.json b/package.json index 5ee43e3c1..7f1476b45 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "docusaurus": "docusaurus", "start": "docusaurus start", "build": "docusaurus build", + "postbuild": "cp build/llms.txt static/llms.txt && cp build/llms-full.txt static/llms-full.txt && ./scripts/clean-llms-imports.sh", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clear": "docusaurus clear", @@ -71,7 +72,8 @@ "sass": "^1.77.8", "styled-jsx": "^5.1.6", "unified": "^11.0.5", - "unist-util-visit": "^5.0.0" + "unist-util-visit": "^5.0.0", + "docusaurus-plugin-llms": "^0.2.2" }, "devDependencies": { "@docusaurus/module-type-aliases": "^3.4.0", diff --git a/scripts/clean-llms-imports.sh b/scripts/clean-llms-imports.sh new file mode 100755 index 000000000..857e32d9d --- /dev/null +++ b/scripts/clean-llms-imports.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Script to clean React/MDX import statements from llms.txt files +# These import statements are extracted by docusaurus-plugin-llms from MDX files +# and need to be removed to keep the files as plain text for LLMs + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +STATIC_DIR="$SCRIPT_DIR/../static" + +echo "Cleaning import statements from llms.txt files..." + +# Clean llms.txt - replace import statements with [Description not available] +if [ -f "$STATIC_DIR/llms.txt" ]; then + sed -i.bak 's/: import [A-Za-z{}, ]* from ['\''"].*['\''"];*$/: [Description not available]/' "$STATIC_DIR/llms.txt" + rm "$STATIC_DIR/llms.txt.bak" + echo "✓ Cleaned $STATIC_DIR/llms.txt" +fi + +# Clean llms-full.txt - remove import statement lines entirely +if [ -f "$STATIC_DIR/llms-full.txt" ]; then + sed -i.bak '/^import [A-Za-z{}, ]* from ['\''"].*['\''"];*$/d' "$STATIC_DIR/llms-full.txt" + rm "$STATIC_DIR/llms-full.txt.bak" + echo "✓ Cleaned $STATIC_DIR/llms-full.txt" +fi + +echo "✓ Import cleanup complete"