feat: Implement automated sitemap generation and add robots.txt with content signals for SEO.

Author: junian

ci-build.sh

@@ -11,3 +11,5 @@ chmod +x dotnet-install.sh
     src/BlazorApps/BlazorApps.csproj \
     -c Release \
     -o publish
+
+./generate-sitemap.sh

generate-sitemap.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ROOT_DIR="publish/wwwroot"
+OUTPUT="$ROOT_DIR/sitemap.xml"
+APPSETTINGS="$ROOT_DIR/appsettings.json"
+
+# Extract BaseUrl from appsettings.json
+BASE_URL=$(grep -o '"BaseUrl"[[:space:]]*:[[:space:]]*"[^"]*"' "$APPSETTINGS" \
+  | sed -E 's/.*"BaseUrl"[[:space:]]*:[[:space:]]*"([^"]*)".*/\1/')
+
+if [[ -z "$BASE_URL" ]]; then
+  echo "BaseUrl not found in appsettings.json"
+  exit 1
+fi
+
+# Normalize base URL (remove trailing slash)
+BASE_URL="${BASE_URL%/}"
+
+echo "Using BaseUrl: $BASE_URL"
+
+# Write XML header
+cat > "$OUTPUT" <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+EOF
+
+# Find index.html files
+find "$ROOT_DIR" -type f -name "index.html" \
+  ! -path "*/_content/*" \
+  ! -path "*/_framework/*" \
+  ! -path "*/css/*" \
+  | while read -r file; do
+
+    # Get directory path relative to wwwroot
+    rel_dir=$(dirname "${file#$ROOT_DIR}")
+
+    # Root index.html → "/"
+    if [[ "$rel_dir" == "" || "$rel_dir" == "/" ]]; then
+      url="$BASE_URL/"
+    else
+      url="$BASE_URL$rel_dir/"
+    fi
+
+    # Get last modified date in ISO 8601
+    lastmod=$(date -u -r "$file" +"%Y-%m-%dT%H:%M:%SZ")
+
+    cat >> "$OUTPUT" <<EOF
+  <url>
+    <loc>$url</loc>
+    <lastmod>$lastmod</lastmod>
+  </url>
+EOF
+done
+
+# Close XML
+cat >> "$OUTPUT" <<EOF
+</urlset>
+EOF
+
+echo "Sitemap generated at $OUTPUT"

src/BlazorApps/wwwroot/appsettings.json

@@ -1,4 +1,5 @@
 {
+    "BaseUrl": "https://lab.junian.net",
     "Title": "Blazor Lab",
     "GoogleAnalytics": {
         "Id": "G-BJ58GF34YZ"

src/BlazorApps/wwwroot/robots.txt

@@ -0,0 +1,30 @@
+# As a condition of accessing this website, you agree to abide by the following
+# content signals:
+
+# (a)  If a content-signal = yes, you may collect content for the corresponding
+#      use.
+# (b)  If a content-signal = no, you may not collect content for the
+#      corresponding use.
+# (c)  If the website operator does not include a content signal for a
+#      corresponding use, the website operator neither grants nor restricts
+#      permission via content signal with respect to the corresponding use.
+
+# The content signals and their meanings are:
+
+# search:   building a search index and providing search results (e.g., returning
+#           hyperlinks and short excerpts from your website's contents). Search does not
+#           include providing AI-generated search summaries.
+# ai-input: inputting content into one or more AI models (e.g., retrieval
+#           augmented generation, grounding, or other real-time taking of content for
+#           generative AI search answers).
+# ai-train: training or fine-tuning AI models.
+
+# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
+# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
+# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.
+
+User-Agent: *
+Content-Signal: ai-train=no, search=yes, ai-input=no
+Allow: /
+
+Sitemap: https://lab.junian.net/sitemap.xml