Allow search engine indexing of the documentation (#948)

* add sitemap

* add robots.txt and configure sitemap

* sphinx-sitemap pip install

* bug

Author: adivijaykumar

.github/workflows/pages.yml

@@ -31,7 +31,9 @@ jobs:
           fetch-depth: 0
           fetch-tags: true
       - name: install
-        run: python -m pip install .
+        run: |
+          python -m pip install .
+          python -m pip install sphinx-sitemap
       - name: Run jupyter notebooks
         run: |
           python -m ipykernel install

docs/conf.py

@@ -66,7 +66,8 @@ def git_upstream_url():
     'sphinx.ext.autosectionlabel',
     'sphinx_tabs.tabs',
     "sphinx.ext.linkcode",
-    'myst_parser'
+    'myst_parser',
+    "sphinx_sitemap",
 ]
 autosummary_generate = True
 
@@ -110,6 +111,12 @@ def git_upstream_url():
 # This patterns also effect to html_static_path and html_extra_path
 exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'requirements.txt']
 
+# The html path allows for google search console verification through
+# Aditya Vijaykumar's gmail ID
+
+html_extra_path = ['robots.txt', 'google063678b5c432c237.html']
+
+
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 
@@ -150,6 +157,7 @@ def git_upstream_url():
     ]
 }
 
+html_baseurl = 'https://bilby-dev.github.io/bilby/'
 
 # -- Options for HTMLHelp output ------------------------------------------
 

docs/google063678b5c432c237.html

@@ -0,0 +1 @@
+google-site-verification: google063678b5c432c237.html

docs/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://bilby-dev.github.io/bilby/sitemap.xml