ENG-9295: add docs llms.txt (#6327)

Author: adhami3310

docs/app/agent_files/__init__.py

@@ -0,0 +1,3 @@
+from ._plugin import AgentFilesPlugin
+
+__all__ = ["AgentFilesPlugin"]

docs/app/agent_files/_plugin.py

@@ -0,0 +1,54 @@
+from collections.abc import Sequence
+from pathlib import Path, PosixPath
+
+from reflex.constants import Dirs
+from reflex_base.plugins import CommonContext, Plugin
+from typing_extensions import Unpack
+
+
+def generate_markdown_files() -> tuple[tuple[Path, str | bytes], ...]:
+    from reflex_docs.pages.docs import doc_markdown_sources
+
+    return tuple(
+        [
+            (PosixPath(route.strip("/") + ".md"), resolved.read_bytes())
+            for route, source_path in doc_markdown_sources.items()
+            if (resolved := Path(source_path)).is_file()
+        ]
+    )
+
+
+def generate_llms_txt(
+    markdown_files: Sequence[tuple[Path, str | bytes]],
+) -> tuple[Path, str]:
+    from reflex_base.config import get_config
+
+    config = get_config()
+
+    if deploy_url := config.deploy_url:
+        deploy_url = config.deploy_url.removesuffix("/")
+    else:
+        deploy_url = ""
+
+    return (
+        Path("docs") / "llms.txt",
+        "# Reflex\n\n"
+        + "## Docs\n\n"
+        + "\n".join(
+            f"- [{deploy_url}/{url}]({deploy_url}/{url})" for url, _ in markdown_files
+        ),
+    )
+
+
+def generate_agent_files() -> tuple[tuple[Path, str | bytes], ...]:
+    markdown_files = generate_markdown_files()
+    return (*markdown_files, generate_llms_txt(markdown_files))
+
+
+class AgentFilesPlugin(Plugin):
+    def get_static_assets(
+        self, **context: Unpack[CommonContext]
+    ) -> Sequence[tuple[Path, str | bytes]]:
+        return [
+            (Dirs.PUBLIC / path, content) for path, content in generate_agent_files()
+        ]

docs/app/reflex_docs/pages/docs/markdown_api.py

@@ -12,14 +12,12 @@
 from reflex_ui_shared.telemetry import get_pixel_website_trackers
 
 from reflex_docs.pages import page404, routes
-from reflex_docs.pages.docs.markdown_api import generate_markdown_files
 from reflex_docs.whitelist import _check_whitelisted_path
 
 # This number discovered by trial and error on Windows 11 w/ Node 18, any
 # higher and the prod build fails with EMFILE error.
 WINDOWS_MAX_ROUTES = int(os.environ.get("REFLEX_WEB_WINDOWS_MAX_ROUTES", "100"))
 
-generate_markdown_files()
 # Create the app.
 app = rxe.App(
     style=styles.BASE_STYLE,

docs/app/reflex_docs/reflex_docs.py

@@ -1,11 +1,17 @@
 import reflex as rx
 
+from agent_files import AgentFilesPlugin
+
 config = rx.Config(
     app_name="reflex_docs",
     deploy_url="https://reflex.dev",
     frontend_packages=[
         "tailwindcss-animated",
     ],
     telemetry_enabled=False,
-    plugins=[rx.plugins.TailwindV4Plugin(), rx.plugins.SitemapPlugin()],
+    plugins=[
+        rx.plugins.TailwindV4Plugin(),
+        rx.plugins.SitemapPlugin(),
+        AgentFilesPlugin(),
+    ],
 )