1688 create sitemapxml and robotstxt (#1690)

krowvin · web-flow · commit 3eaf8e966178 · 2026-04-14T13:17:42.000-05:00
React router now uses a small shared route manifest so the GUI build can
generate
sitemap.xml automatically from the app routes. 

i.e. `cda-gui/src/route-paths.js`

Sitemap is packaged into the WAR and also copied to Tomcat ROOT in the
Docker image so it’s available at /sitemap.xml.
Tomcat root app was updated to replace the placeholder page. / now
returns a real HTTP 302 redirect to /cwms-data/, and /robots.txt is
served from root with rules that allow the GUI while blocking raw
API/query surfaces.

Confirmed war builds with sitemap.xml in it

Verified the container root returns 302 Location: /cwms-data/
diff --git a/Dockerfile b/Dockerfile
@@ -33,11 +33,13 @@ RUN mkdir /download && \
     rm -rf /download && \
     rm -rf /usr/local/tomcat/webapps/* && \
     mkdir /usr/local/tomcat/webapps/ROOT && \
-    echo "<html><body>Nothing to see here</body></html>" > /usr/local/tomcat/webapps/ROOT/index.html
+    printf "<%% response.sendRedirect(\"/cwms-data/\"); %%>\n" > /usr/local/tomcat/webapps/ROOT/index.jsp && \
+    printf "User-agent: *\nAllow: /cwms-data/\nDisallow: /cwms-data/auth/\nDisallow: /cwms-data/catalog/\nDisallow: /cwms-data/timeseries/\nDisallow: /cwms-data/swagger-docs\nDisallow: /auth/\nSitemap: https://cwms-data.usace.army.mil/sitemap.xml\n" > /usr/local/tomcat/webapps/ROOT/robots.txt
 CMD ["/usr/local/tomcat/bin/catalina.sh","run"]
 
 FROM tomcat_base AS api
 
+COPY --from=builder /builddir/cda-gui/dist/sitemap.xml /usr/local/tomcat/webapps/ROOT/sitemap.xml
 COPY --from=builder /builddir/cwms-data-api/build/docker/cda/ /usr/local/tomcat
 COPY --from=builder /builddir/cwms-data-api/build/docker/context.xml /usr/local/tomcat/conf
 COPY --from=builder /builddir/cwms-data-api/build/docker/server.xml /usr/local/tomcat/conf
diff --git a/cda-gui/build.gradle b/cda-gui/build.gradle
@@ -7,8 +7,10 @@ task buildGui(type:NpxTask) {
     dependsOn npmInstall
     command = "npm"
     args = ["run", "build", "--prod"]
-    inputs.files('package.json', 'package-lock.json', 'vite.config.js')
+    inputs.files('package.json', 'package-lock.json', 'vite.config.js', 'index.html')
     inputs.dir('src')
+    inputs.dir('public')
+    inputs.dir('scripts')
     inputs.dir(fileTree("node_modules").exclude(".cache"))
     outputs.dir('dist')
 }
@@ -38,4 +40,4 @@ sourceSets {
             }
         }
     }
-}
+}
diff --git a/cda-gui/package.json b/cda-gui/package.json
@@ -5,7 +5,7 @@
   "type": "module",
   "scripts": {
     "dev": "vite",
-    "build": "vite build",
+    "build": "vite build && node scripts/generate-sitemap.mjs",
     "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
     "preview": "vite preview",
     "prepare": "cd .. && husky cda-gui/.husky",
diff --git a/cda-gui/scripts/generate-sitemap.mjs b/cda-gui/scripts/generate-sitemap.mjs
@@ -0,0 +1,36 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { sitemapPaths } from "../src/route-paths.js";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const projectDir = path.resolve(__dirname, "..");
+const outputPath = path.join(projectDir, "dist", "sitemap.xml");
+
+const siteOrigin = (process.env.SITE_ORIGIN ?? "https://cwms-data.usace.army.mil").replace(
+  /\/+$/,
+  "",
+);
+const siteBasePath = (process.env.SITE_BASE_PATH ?? "/cwms-data").replace(/\/+$/, "");
+
+const urls = sitemapPaths.map((routePath) => {
+  const normalizedPath = routePath ? `/${routePath}` : "";
+  return `${siteOrigin}${siteBasePath}${normalizedPath}`;
+});
+
+const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${urls
+  .map(
+    (url) => `  <url>
+    <loc>${url}</loc>
+  </url>`,
+  )
+  .join("\n")}
+</urlset>
+`;
+
+await mkdir(path.dirname(outputPath), { recursive: true });
+await writeFile(outputPath, xml, "utf8");
diff --git a/cda-gui/src/main.jsx b/cda-gui/src/main.jsx
@@ -21,8 +21,18 @@ import ErrorFallback from "./pages/ErrorFallback";
 import FilterExpressions from "./pages/rsql";
 import Timestamps from "./pages/timestamps";
 import LegacyFormat from "./pages/legacy-format/index.jsx";
+import { routePaths } from "./route-paths";
 
 const queryClient = new QueryClient();
+const routeComponents = {
+  home: Home,
+  "swagger-ui": SwaggerUI,
+  "data-query": DataQuery,
+  regexp: Regexp,
+  "filter-expressions": FilterExpressions,
+  timestamps: Timestamps,
+  "legacy-format": LegacyFormat,
+};
 
 const router = createBrowserRouter(
   [
@@ -31,16 +41,12 @@ const router = createBrowserRouter(
       element: <Layout />,
       errorElement: <ErrorFallback />,
       children: [
-        { index: true, element: <Home /> },
-        {
-          path: "swagger-ui",
-          element: <SwaggerUI />,
-        },
-        { path: "data-query", element: <DataQuery /> },
-        { path: "regexp", element: <Regexp /> },
-        { path: "filter-expressions", element: <FilterExpressions /> },
-        { path: "timestamps", element: <Timestamps /> },
-        { path: "legacy-format", element: <LegacyFormat /> },
+        ...routePaths.map(({ id, index, path }) => {
+          const Component = routeComponents[id];
+          return index
+            ? { index: true, element: <Component /> }
+            : { path, element: <Component /> };
+        }),
         { path: "*", element: <NotFound /> },
       ],
     },
diff --git a/cda-gui/src/route-paths.js b/cda-gui/src/route-paths.js
@@ -0,0 +1,40 @@
+// Routes are defined here to allow building a sitemap dynamically
+export const routePaths = [
+  {
+    id: "home",
+    index: true,
+    sitemapPath: "",
+  },
+  {
+    id: "swagger-ui",
+    path: "swagger-ui",
+    sitemapPath: "swagger-ui",
+  },
+  {
+    id: "data-query",
+    path: "data-query",
+    sitemapPath: "data-query",
+  },
+  {
+    id: "regexp",
+    path: "regexp",
+    sitemapPath: "regexp",
+  },
+  {
+    id: "filter-expressions",
+    path: "filter-expressions",
+    sitemapPath: "filter-expressions",
+  },
+  {
+    id: "timestamps",
+    path: "timestamps",
+    sitemapPath: "timestamps",
+  },
+  {
+    id: "legacy-format",
+    path: "legacy-format",
+    sitemapPath: "legacy-format",
+  },
+];
+
+export const sitemapPaths = routePaths.map(({ sitemapPath }) => sitemapPath);

Original file line number	Diff line number	Diff line change
`@@ -7,8 +7,10 @@ task buildGui(type:NpxTask) {`
`7`	`7`	`dependsOn npmInstall`
`8`	`8`	`command = "npm"`
`9`	`9`	`args = ["run", "build", "--prod"]`
`10`		`- inputs.files('package.json', 'package-lock.json', 'vite.config.js')`
	`10`	`+ inputs.files('package.json', 'package-lock.json', 'vite.config.js', 'index.html')`
`11`	`11`	`inputs.dir('src')`
	`12`	`+ inputs.dir('public')`
	`13`	`+ inputs.dir('scripts')`
`12`	`14`	`inputs.dir(fileTree("node_modules").exclude(".cache"))`
`13`	`15`	`outputs.dir('dist')`
`14`	`16`	`}`
`@@ -38,4 +40,4 @@ sourceSets {`
`38`	`40`	`}`
`39`	`41`	`}`
`40`	`42`	`}`
`41`		`-}`
	`43`	`+}`