Add a robots.txt that only allows crawl of the latest English documentation.

fdxmw · fdxmw · commit 1be7976c195b · 2026-03-13T14:05:33.000-07:00
This discourages crawl and index for other versions of the documentation.
Search engines often direct users to the other versions, which are usually not
what the user is looking for.
diff --git a/docs/conf.py b/docs/conf.py
@@ -73,6 +73,7 @@
 # a list of builtin themes.
 #
 html_baseurl = "https://pyrtl.readthedocs.io/en/latest"
+html_extra_path = ["html_root"]
 html_theme = "furo"
 html_theme_options = {
     "sidebar_hide_name": True,
diff --git a/docs/html_root/robots.txt b/docs/html_root/robots.txt
@@ -0,0 +1,6 @@
+# Only crawl the latest English documentation.
+User-agent: *
+Disallow: /
+Allow: /en/latest/
+
+Sitemap: https://pyrtl.readthedocs.io/sitemap.xml

Original file line number	Diff line number	Diff line change
`@@ -73,6 +73,7 @@`
`73`	`73`	`# a list of builtin themes.`
`74`	`74`	`#`
`75`	`75`	`html_baseurl = "https://pyrtl.readthedocs.io/en/latest"`
	`76`	`+html_extra_path = ["html_root"]`
`76`	`77`	`html_theme = "furo"`
`77`	`78`	`html_theme_options = {`
`78`	`79`	`"sidebar_hide_name": True,`