feat(seo): add noindex handling and robots.txt deployment controls

Agent-Logs-Url: https://github.com/nextcloud/documentation/sessions/94393687-97e6-46db-ae6e-523576c970ab

Co-authored-by: skjnldsv <14975046+skjnldsv@users.noreply.github.com>

Author: 2 people

.github/workflows/sphinxbuild.yml

@@ -387,6 +387,7 @@ jobs:
         run: |
           find "stage/${{ needs.stage-and-check.outputs.branch_name }}" -name '*.html' -print0 | while IFS= read -r -d '' f; do
             perl -0pi -e 's{^\s*<link rel="canonical" href="https://docs\.nextcloud\.com/server/[^"]*" />\n}{}m' "$f"
+            perl -pi -e 's{<meta name="robots" content="noindex, follow" />\n?}{}g' "$f"
           done
           ls -la stage/*
 
@@ -504,6 +505,24 @@ jobs:
             echo "has_changes=true" >> $GITHUB_OUTPUT
           fi
 
+      - name: Strip noindex from stable docs
+        run: |
+          # Strip noindex from stable docs – these are the canonical pages we want indexed
+          if [ -d server/stable ]; then
+            find server/stable -name '*.html' -print0 | \
+              xargs -0 perl -pi -e 's{<meta name="robots" content="noindex, follow" />\n?}{}g'
+          fi
+
+      - name: Write robots.txt
+        run: |
+          cat > robots.txt << 'EOF'
+          User-agent: *
+          # Only the stable version should be indexed
+          Allow: /server/stable/
+          Disallow: /server/
+          Sitemap: https://docs.nextcloud.com/sitemap.xml
+          EOF
+
         # Remove the stage/ directory BEFORE creating the PR so it doesn't get committed
       - name: Clean up staging cache before commit
         run: rm -rf stage/

_shared_assets/templates/layout.html

@@ -0,0 +1,5 @@
+{% extends "!layout.html" %}
+{% block extrahead %}
+  {{ super() }}
+  <meta name="robots" content="noindex, follow" />
+{% endblock %}