Add llms.txt and llms-full.txt generation

Author: AdrianAtZyte

.github/workflows/main.yml

@@ -43,6 +43,11 @@ jobs:
           - python-version: "3.13"
             env:
               TOXENV: "json"
+          # Keep in sync with pyproject.toml's
+          # tool.sphinx-scrapy.python-version.
+          - python-version: "3.14"
+            env:
+              TOXENV: "docs"
 
           # https://github.com/kevin1024/vcrpy/issues/885
           # - python-version: "pypy3.11-v7.3.20"

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+repos:
+- repo: https://github.com/scrapy/sphinx-scrapy
+  rev: 999017c29b8bc213843e86e0e350a85c47493e34
+  hooks:
+    - id: sphinx-scrapy

.readthedocs.yml

@@ -1,15 +1,9 @@
 version: 2
-formats: all
-sphinx:
-  configuration: docs/conf.py
-  fail_on_warning: true
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
-    # For available versions, see:
-    # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
-    python: "3.10"  # Keep in sync with .github/workflows/main.yml
-python:
-  install:
-    - requirements: docs/requirements.txt
-    - path: .
+    python: "3.14"
+  commands:
+    - pip install tox
+    - tox -e docs
+    - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/

docs/Makefile

@@ -39,7 +39,7 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.autodoc']
+extensions = ['sphinx_scrapy']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -160,9 +160,3 @@
      author, 'python-scrapinghub', 'One line description of project.',
      'Miscellaneous'),
 ]
-
-# Set Sphinx Read The Docs theme
-import sphinx_rtd_theme
-
-html_theme = 'sphinx_rtd_theme'
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

docs/conf.py

@@ -4,7 +4,7 @@ Quickstart
 Requirements
 ------------
 
-* Python 3.8+
+* Python 3.9+
 
 
 Installation

docs/make.bat

@@ -0,0 +1,2 @@
+sphinx-rtd-theme
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34

docs/quickstart.rst

@@ -1,2 +1,68 @@
-sphinx==7.2.6
-sphinx-rtd-theme==2.0.0
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements.in
+alabaster==1.0.0
+    # via sphinx
+babel==2.18.0
+    # via sphinx
+certifi==2026.2.25
+    # via requests
+charset-normalizer==3.4.6
+    # via requests
+docutils==0.22.4
+    # via
+    #   sphinx
+    #   sphinx-markdown-builder
+    #   sphinx-rtd-theme
+idna==3.11
+    # via requests
+imagesize==2.0.0
+    # via sphinx
+jinja2==3.1.6
+    # via sphinx
+markupsafe==3.0.3
+    # via jinja2
+packaging==26.0
+    # via
+    #   sphinx
+    #   sphinx-scrapy
+pygments==2.19.2
+    # via sphinx
+requests==2.32.5
+    # via sphinx
+roman-numerals==4.1.0
+    # via sphinx
+snowballstemmer==3.0.1
+    # via sphinx
+sphinx==9.1.0
+    # via
+    #   sphinx-llms-txt
+    #   sphinx-markdown-builder
+    #   sphinx-rtd-theme
+    #   sphinx-scrapy
+    #   sphinxcontrib-jquery
+sphinx-llms-txt==0.7.1
+    # via sphinx-scrapy
+sphinx-markdown-builder @ git+https://github.com/AdrianAtZyte/sphinx-markdown-builder.git@2050ffaed43fcc523e03cc7a677cbcb5f5f6d30b
+    # via sphinx-scrapy
+sphinx-rtd-theme==3.1.0
+    # via -r requirements.in
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34
+    # via -r requirements.in
+sphinxcontrib-applehelp==2.0.0
+    # via sphinx
+sphinxcontrib-devhelp==2.0.0
+    # via sphinx
+sphinxcontrib-htmlhelp==2.1.0
+    # via sphinx
+sphinxcontrib-jquery==4.1
+    # via sphinx-rtd-theme
+sphinxcontrib-jsmath==1.0.1
+    # via sphinx
+sphinxcontrib-qthelp==2.0.0
+    # via sphinx
+sphinxcontrib-serializinghtml==2.0.0
+    # via sphinx
+tabulate==0.10.0
+    # via sphinx-markdown-builder
+urllib3==2.6.3
+    # via requests

docs/requirements.in

@@ -0,0 +1,3 @@
+[tool.sphinx-scrapy]
+# Keep in sync with .github/workflows/main.yml
+python-version = "3.14"