Add llms.txt docs (#2925)

## Summary
Adds [`sphinx-llms-txt`](https://github.com/jdillard/sphinx-llms-txt) to
the Sphinx documentation build so that `llms.txt` and `llms-full.txt`
are generated automatically alongside the existing HTML output.

[`llms.txt`](https://llmstxt.org) is a standard for making documentation
consumable by LLMs — `llms.txt` is a structured index of doc pages with
links, and `llms-full.txt` is the full documentation concatenated into a
single plain-text file.
Once merged, the files will be served at:
 - https://sttp.softwaremill.com/en/latest/llms.txt
 - https://sttp.softwaremill.com/en/latest/llms-full.txt

## Changes
- `doc/requirements.txt` / `generated-doc/out/requirements.txt` — add
`sphinx-llms-txt`
 - `doc/conf.py` / `generated-doc/out/conf.py`:
- Add `sphinx_llms_txt` extension with title, summary, and
`llms-full.txt` enabled
- Remove `.txt` from `source_suffix` (was causing `requirements.txt` to
appear as a doc page)
- Tighten `exclude_patterns` to prevent venv, node_modules,
`_templates`, `includes/*` partials, and `requirements.txt` from being
treated as source documents
- Add fallback to `html_baseurl` so local builds also produce absolute
links

No changes to `.readthedocs.yaml` - ReadTheDocs already installs from
`generated-doc/out/requirements.txt` and runs Sphinx on
`generated-doc/out/conf.py`, so the new files will be generated on every
ReadTheDocs build automatically.

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: flsh86

.gitignore

@@ -42,3 +42,7 @@ work.txt
 # TraceVault — local only, do not commit
 .tracevault/
 .claude/settings.local.json
+
+# Local Sphinx build artifacts
+generated-docs/out/.venv/
+generated-docs/out/_build/

docs/conf.py

@@ -24,7 +24,10 @@
 import os
 
 # Define the canonical URL if you are using a custom domain on Read the Docs
-html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "")
+html_baseurl = os.environ.get(
+    "READTHEDOCS_CANONICAL_URL",
+    "https://sttp.softwaremill.com/en/latest/",
+)
 
 # Tell Jinja2 templates the build is running on Read the Docs
 if os.environ.get("READTHEDOCS", "") == "True":
@@ -41,10 +44,14 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['myst_parser', 'sphinx_rtd_theme']
+extensions = ['myst_parser', 'sphinx_rtd_theme', 'sphinx_llms_txt']
 
 myst_enable_extensions = ['attrs_block']
 
+llms_txt_title = "sttp"
+llms_txt_summary = "The Scala HTTP client you always wanted!"
+llms_txt_full_file = True
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -54,7 +61,6 @@
 # source_suffix = ['.rst', '.md']
 source_suffix = {
     '.rst': 'restructuredtext',
-    '.txt': 'markdown',
     '.md': 'markdown',
 }
 
@@ -85,7 +91,15 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = [
+    '_build', 'Thumbs.db', '.DS_Store',
+    '.venv', 'venv', 'env',
+    '**/site-packages/**',
+    '**/node_modules/**',
+    '_templates',
+    'requirements.txt',
+    'includes/*',
+]
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'default'

docs/requirements.txt

@@ -2,3 +2,4 @@ sphinx_rtd_theme==2.0.0
 sphinx==7.3.7
 sphinx-autobuild==2024.4.16
 myst-parser==2.0.0
+sphinx-llms-txt

generated-docs/out/conf.py

@@ -24,7 +24,10 @@
 import os
 
 # Define the canonical URL if you are using a custom domain on Read the Docs
-html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "")
+html_baseurl = os.environ.get(
+    "READTHEDOCS_CANONICAL_URL",
+    "https://sttp.softwaremill.com/en/latest/",
+)
 
 # Tell Jinja2 templates the build is running on Read the Docs
 if os.environ.get("READTHEDOCS", "") == "True":
@@ -41,10 +44,14 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['myst_parser', 'sphinx_rtd_theme']
+extensions = ['myst_parser', 'sphinx_rtd_theme', 'sphinx_llms_txt']
 
 myst_enable_extensions = ['attrs_block']
 
+llms_txt_title = "sttp"
+llms_txt_summary = "The Scala HTTP client you always wanted!"
+llms_txt_full_file = True
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -54,7 +61,6 @@
 # source_suffix = ['.rst', '.md']
 source_suffix = {
     '.rst': 'restructuredtext',
-    '.txt': 'markdown',
     '.md': 'markdown',
 }
 
@@ -85,7 +91,15 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = [
+    '_build', 'Thumbs.db', '.DS_Store',
+    '.venv', 'venv', 'env',
+    '**/site-packages/**',
+    '**/node_modules/**',
+    '_templates',
+    'requirements.txt',
+    'includes/*',
+]
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'default'

generated-docs/out/requirements.txt

@@ -2,3 +2,4 @@ sphinx_rtd_theme==2.0.0
 sphinx==7.3.7
 sphinx-autobuild==2024.4.16
 myst-parser==2.0.0
+sphinx-llms-txt