Merge pull request #117 from Point72/tkp/llms

feat(llms): add llms.txt generation via sphinx-llm

Author: timkpaine

pyproject.toml

@@ -60,6 +60,9 @@ wiki = [
     "sphinx-markdown-builder>=0.6.9",
 
 ]
+llms = [
+    "sphinx-llm>=0.4.1",
+]
 themes = [
     "shibuya",
     "sphinxawesome-theme",
@@ -84,6 +87,7 @@ develop = [
     "sphinx-rust",
     "sphinx-js>=5.0.0",
     "sphinx-markdown-builder>=0.6.9",
+    "sphinx-llm>=0.4.1",
     # Themes
     "shibuya",
     "sphinxawesome-theme",
@@ -223,3 +227,10 @@ generate-footer = true
 footer-docs-url = "https://yardang.python-templates.dev"
 footer-repo-url = "https://github.com/python-project-templates/yardang"
 markdown-flavor = "github"
+
+[tool.yardang.llms]
+enabled = true
+description = "Easily generate sphinx documentation"
+build-parallel = true
+suffix-mode = "auto"
+full-build = true

yardang/build.py

@@ -520,6 +520,31 @@ def customize(args):
         # Determine if wiki/markdown output should be generated
         use_wiki = wiki_args["wiki_enabled"]
 
+        # Load sphinx-llm (llms.txt) configuration from tool.yardang.llms
+        llms_config_base = f"{config_base}.llms"
+        llms_args = {}
+        for config_option, default in {
+            # yardang gate
+            "llms_enabled": False,
+            # sphinx-llm passthrough options
+            "llms_txt_description": "",
+            "llms_txt_build_parallel": True,
+            "llms_txt_suffix_mode": "auto",
+            "llms_txt_full_build": True,
+        }.items():
+            # config keys in toml use hyphens, not underscores, and drop the
+            # llms_/llms_txt_ prefix
+            if config_option.startswith("llms_txt_"):
+                toml_key = config_option.replace("llms_txt_", "").replace("_", "-")
+            else:
+                toml_key = config_option.replace("llms_", "").replace("_", "-")
+            llms_args[config_option] = get_config(section=toml_key, base=llms_config_base)
+            if llms_args[config_option] is None:
+                llms_args[config_option] = default
+
+        # Determine if llms.txt output should be generated
+        use_llms = llms_args["llms_enabled"]
+
         # create a temporary directory to store the conf.py file in
         with TemporaryDirectory() as td:
             templateEnv = Environment(loader=FileSystemLoader(searchpath=str(Path(__file__).parent.resolve())))
@@ -545,10 +570,12 @@ def customize(args):
                 use_sphinx_rust=use_sphinx_rust,
                 use_sphinx_js=use_sphinx_js,
                 use_wiki=use_wiki,
+                use_llms=use_llms,
                 **breathe_args,
                 **rust_args,
                 **js_args,
                 **wiki_args,
+                **llms_args,
                 **configuration_args,
             )
 

yardang/conf.py.j2

@@ -45,6 +45,7 @@ use_breathe = {{use_breathe}}  # noqa: F821
 use_sphinx_rust = {{use_sphinx_rust}}  # noqa: F821
 use_sphinx_js = {{use_sphinx_js}}  # noqa: F821
 use_wiki = {{use_wiki}}  # noqa: F821
+use_llms = {{use_llms}}  # noqa: F821
 
 #############################################################################################################
 #  _____                        _                           _ _  __            _          _                 #
@@ -89,6 +90,10 @@ if use_sphinx_js:
 if use_wiki:
     extensions.append("sphinx_markdown_builder")
 
+# Add sphinx-llm extension if llms.txt generation is enabled
+if use_llms:
+    extensions.append("sphinx_llm.txt")
+
 if use_autoapi in (True, None):
     # add if it is set to true or if it is set to None
     # NOTE: bug in autoapi that requires
@@ -232,6 +237,15 @@ if use_wiki:
     markdown_bullet = "{{markdown_bullet}}"
     markdown_flavor = "{{markdown_flavor}}"
 
+# sphinx-llm (llms.txt) configuration
+if use_llms:
+    {% if llms_txt_description %}
+    llms_txt_description = """{{llms_txt_description}}"""
+    {% endif %}
+    llms_txt_build_parallel = {{llms_txt_build_parallel}}
+    llms_txt_suffix_mode = "{{llms_txt_suffix_mode}}"
+    llms_txt_full_build = {{llms_txt_full_build}}
+
 # autosummary
 autosummary_generate = True  # if using autosummary, autogenerate
 

yardang/tests/test_llms.py

@@ -0,0 +1,146 @@
+"""Tests for llms.txt generation (sphinx-llm) in yardang."""
+
+import os
+from pathlib import Path
+
+
+class TestLlmsConfiguration:
+    """Tests for llms.txt configuration loading and generation."""
+
+    def test_llms_config_loading_from_pyproject(self, tmp_path):
+        """Test that llms configuration is loaded from pyproject.toml."""
+        pyproject_content = """
+[project]
+name = "test-project"
+version = "1.0.0"
+
+[tool.yardang]
+title = "Test Project"
+root = "README.md"
+use-autoapi = false
+
+[tool.yardang.llms]
+enabled = true
+description = "A project for LLMs"
+build-parallel = false
+suffix-mode = "replace"
+full-build = false
+"""
+        pyproject_path = tmp_path / "pyproject.toml"
+        pyproject_path.write_text(pyproject_content)
+
+        readme_path = tmp_path / "README.md"
+        readme_path.write_text("# Test Project\n\nTest content.")
+
+        original_cwd = os.getcwd()
+        try:
+            os.chdir(tmp_path)
+
+            from yardang.utils import get_config
+
+            assert get_config(section="enabled", base="tool.yardang.llms") is True
+            assert get_config(section="description", base="tool.yardang.llms") == "A project for LLMs"
+            assert get_config(section="build-parallel", base="tool.yardang.llms") is False
+            assert get_config(section="suffix-mode", base="tool.yardang.llms") == "replace"
+            assert get_config(section="full-build", base="tool.yardang.llms") is False
+        finally:
+            os.chdir(original_cwd)
+
+    def test_llms_config_defaults(self, tmp_path):
+        """Test that llms configuration returns None when not specified."""
+        pyproject_content = """
+[project]
+name = "test-project"
+version = "1.0.0"
+
+[tool.yardang]
+title = "Test Project"
+root = "README.md"
+"""
+        pyproject_path = tmp_path / "pyproject.toml"
+        pyproject_path.write_text(pyproject_content)
+
+        readme_path = tmp_path / "README.md"
+        readme_path.write_text("# Test Project\n\nTest content.")
+
+        original_cwd = os.getcwd()
+        try:
+            os.chdir(tmp_path)
+
+            from yardang.utils import get_config
+
+            assert get_config(section="enabled", base="tool.yardang.llms") is None
+            assert get_config(section="suffix-mode", base="tool.yardang.llms") is None
+        finally:
+            os.chdir(original_cwd)
+
+    def test_generate_docs_with_llms_enabled(self, tmp_path):
+        """Test that generate_docs_configuration wires in the sphinx-llm extension."""
+        pyproject_content = """
+[project]
+name = "test-project"
+version = "1.0.0"
+
+[tool.yardang]
+title = "Test Project"
+root = "README.md"
+use-autoapi = false
+
+[tool.yardang.llms]
+enabled = true
+description = "A project for LLMs"
+suffix-mode = "replace"
+"""
+        pyproject_path = tmp_path / "pyproject.toml"
+        pyproject_path.write_text(pyproject_content)
+
+        readme_path = tmp_path / "README.md"
+        readme_path.write_text("# Test Project\n\nTest content.")
+
+        original_cwd = os.getcwd()
+        try:
+            os.chdir(tmp_path)
+
+            from yardang.build import generate_docs_configuration
+
+            with generate_docs_configuration() as conf_dir:
+                conf_content = (Path(conf_dir) / "conf.py").read_text()
+
+                assert "use_llms = True" in conf_content
+                assert 'extensions.append("sphinx_llm.txt")' in conf_content
+                assert "llms_txt_suffix_mode" in conf_content
+                assert "replace" in conf_content
+                assert "A project for LLMs" in conf_content
+        finally:
+            os.chdir(original_cwd)
+
+    def test_generate_docs_llms_disabled_by_default(self, tmp_path):
+        """Test that llms.txt generation is off unless explicitly enabled."""
+        pyproject_content = """
+[project]
+name = "test-project"
+version = "1.0.0"
+
+[tool.yardang]
+title = "Test Project"
+root = "README.md"
+use-autoapi = false
+"""
+        pyproject_path = tmp_path / "pyproject.toml"
+        pyproject_path.write_text(pyproject_content)
+
+        readme_path = tmp_path / "README.md"
+        readme_path.write_text("# Test Project\n\nTest content.")
+
+        original_cwd = os.getcwd()
+        try:
+            os.chdir(tmp_path)
+
+            from yardang.build import generate_docs_configuration
+
+            with generate_docs_configuration() as conf_dir:
+                conf_content = (Path(conf_dir) / "conf.py").read_text()
+
+                assert "use_llms = False" in conf_content
+        finally:
+            os.chdir(original_cwd)