feat(smoke-tests): validate frontend plugins via bundle checks and loaded-plugin probe

Add two-layer frontend plugin validation to the smoke test harness:
- Layer 1: static validation of dist-scalprum/ after OCI download
- Layer 2: inline backend probe plugin querying dynamicPluginsServiceRef

Made-with: Cursor

Author: rostalan

smoke-tests/smoke-test.mjs

@@ -7,6 +7,7 @@ import {
   mkdirSync,
   existsSync,
   rmSync,
+  readdirSync,
 } from "node:fs";
 import { join, resolve, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -88,6 +89,12 @@ function parseOciRef(packageStr) {
   };
 }
 
+const FRONTEND_ROLES = new Set(["frontend-plugin", "frontend-plugin-module"]);
+
+function isFrontendRole(role) {
+  return FRONTEND_ROLES.has(role);
+}
+
 // ---------------------------------------------------------------------------
 // OCI Download  (mirrors install-dynamic-plugins.py §663-715)
 // ---------------------------------------------------------------------------
@@ -148,6 +155,61 @@ async function downloadPlugins(plugins, dest) {
   rmSync(tmpDir, { recursive: true, force: true });
 }
 
+// ---------------------------------------------------------------------------
+// Frontend bundle validation (Layer 1)
+// ---------------------------------------------------------------------------
+
+function findJsFiles(dir) {
+  const entries = readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (findJsFiles(full)) return true;
+    } else if (/\.(js|mjs|cjs)$/.test(entry.name)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function validateFrontendBundles(plugins, pluginsRoot) {
+  const results = [];
+  for (const plugin of plugins) {
+    const { pluginPath } = parseOciRef(plugin.package);
+    if (!pluginPath) continue;
+
+    const { pkgName, role } = readPluginMeta(pluginsRoot, pluginPath);
+    if (!isFrontendRole(role)) continue;
+
+    const scalprumDir = join(pluginsRoot, pluginPath, "dist-scalprum");
+
+    if (!existsSync(scalprumDir)) {
+      results.push({
+        pkgName,
+        role,
+        pluginPath,
+        status: "fail-bundle",
+        detail: "dist-scalprum/ directory missing",
+      });
+      continue;
+    }
+
+    if (!findJsFiles(scalprumDir)) {
+      results.push({
+        pkgName,
+        role,
+        pluginPath,
+        status: "fail-bundle",
+        detail: "dist-scalprum/ contains no .js/.mjs/.cjs files",
+      });
+      continue;
+    }
+
+    results.push({ pkgName, role, pluginPath, status: "pass" });
+  }
+  return results;
+}
+
 // ---------------------------------------------------------------------------
 // Config generation
 // ---------------------------------------------------------------------------
@@ -187,9 +249,10 @@ async function bootBackend(configPaths) {
     dynamicPluginsFeatureLoader,
     CommonJSModuleLoader,
     dynamicPluginsFrontendServiceRef,
+    dynamicPluginsServiceRef,
   } = await import("@backstage/backend-dynamic-feature-service");
   const { PackageRoles } = await import("@backstage/cli-node");
-  const { createServiceFactory } =
+  const { createServiceFactory, createBackendPlugin, coreServices } =
     await import("@backstage/backend-plugin-api");
   const path = await import("node:path");
 
@@ -218,6 +281,36 @@ async function bootBackend(configPaths) {
     }),
   );
 
+  backend.add(
+    createBackendPlugin({
+      pluginId: "smoke-test-probe",
+      register(env) {
+        env.registerInit({
+          deps: {
+            http: coreServices.httpRouter,
+            dynamicPlugins: dynamicPluginsServiceRef,
+          },
+          async init({ http, dynamicPlugins }) {
+            const { Router } = await import("express");
+            const router = Router();
+            router.get("/loaded-plugins", (_, res) => {
+              res.json(dynamicPlugins.plugins({ includeFailed: true }));
+            });
+            http.use(router);
+            try {
+              http.addAuthPolicy({
+                path: "/loaded-plugins",
+                allow: "unauthenticated",
+              });
+            } catch {
+              /* API may not exist on this version */
+            }
+          },
+        });
+      },
+    }),
+  );
+
   backend.add(import("@backstage/plugin-catalog-backend"));
   backend.add(
     import("@backstage/plugin-catalog-backend-module-scaffolder-entity-model"),
@@ -281,6 +374,8 @@ async function probePluginRoutes(plugins, port, pluginsRoot) {
 
     const { pkgName, role, pluginId } = readPluginMeta(pluginsRoot, pluginPath);
 
+    if (isFrontendRole(role)) continue;
+
     if (role !== "backend-plugin") {
       results.push({ pkgName, role, pluginPath, status: "skip" });
       continue;
@@ -322,6 +417,98 @@ async function probePluginRoutes(plugins, port, pluginsRoot) {
   return results;
 }
 
+// ---------------------------------------------------------------------------
+// Frontend plugin probing (Layer 2)
+// ---------------------------------------------------------------------------
+
+async function probeFrontendPlugins(plugins, port, pluginsRoot) {
+  const frontendPlugins = [];
+  for (const plugin of plugins) {
+    const { pluginPath } = parseOciRef(plugin.package);
+    if (!pluginPath) continue;
+    const meta = readPluginMeta(pluginsRoot, pluginPath);
+    if (isFrontendRole(meta.role)) {
+      frontendPlugins.push({ ...meta, pluginPath });
+    }
+  }
+
+  if (frontendPlugins.length === 0) return [];
+
+  const failAll = (detail) =>
+    frontendPlugins.map((fp) => ({
+      pkgName: fp.pkgName,
+      role: fp.role,
+      pluginPath: fp.pluginPath,
+      status: "fail-load",
+      detail,
+    }));
+
+  let res;
+  try {
+    res = await fetch(
+      `http://localhost:${port}/api/smoke-test-probe/loaded-plugins`,
+    );
+  } catch (err) {
+    return failAll(`probe endpoint unreachable: ${err.message}`);
+  }
+
+  if (!res.ok) {
+    return failAll(`probe returned HTTP ${res.status}`);
+  }
+
+  let body;
+  try {
+    body = await res.json();
+  } catch {
+    return failAll("invalid probe response");
+  }
+
+  if (!Array.isArray(body)) {
+    return failAll("invalid probe response");
+  }
+
+  const results = [];
+  for (const fp of frontendPlugins) {
+    const loaded = body.find(
+      (lp) => lp && typeof lp === "object" && lp.name === fp.pkgName,
+    );
+
+    if (!loaded) {
+      results.push({
+        pkgName: fp.pkgName,
+        role: fp.role,
+        pluginPath: fp.pluginPath,
+        status: "fail-load",
+        detail: "not found in loaded plugins list",
+      });
+    } else if (loaded.platform !== "web") {
+      results.push({
+        pkgName: fp.pkgName,
+        role: fp.role,
+        pluginPath: fp.pluginPath,
+        status: "fail-load",
+        detail: `unexpected platform: ${loaded.platform}`,
+      });
+    } else if (loaded.failure) {
+      results.push({
+        pkgName: fp.pkgName,
+        role: fp.role,
+        pluginPath: fp.pluginPath,
+        status: "fail-load",
+        detail: `plugin loaded with failure: ${loaded.failure}`,
+      });
+    } else {
+      results.push({
+        pkgName: fp.pkgName,
+        role: fp.role,
+        pluginPath: fp.pluginPath,
+        status: "pass",
+      });
+    }
+  }
+  return results;
+}
+
 // ---------------------------------------------------------------------------
 // Reporting
 // ---------------------------------------------------------------------------
@@ -336,33 +523,86 @@ function reportAndWrite(results, resultsFile) {
         console.log(`  SKIP  ${r.pkgName}  (${r.role})`);
         break;
       case "pass":
-        console.log(`  PASS  ${r.pkgName}  → /api/${r.pluginId}  (${r.http})`);
+        if (r.pluginId) {
+          console.log(
+            `  PASS  ${r.pkgName}  → /api/${r.pluginId}  (${r.http})`,
+          );
+        } else if (isFrontendRole(r.role)) {
+          console.log(`  PASS  ${r.pkgName}  (${r.role})`);
+        }
         break;
       case "warn":
         console.log(
           `  WARN  ${r.pkgName}  → /api/${r.pluginId}  (404 — pluginId guess may be wrong)`,
         );
         break;
+      case "fail-bundle":
+        console.log(`  FAIL  ${r.pkgName}  [bundle] ${r.detail}`);
+        failedPlugins.push(r.pkgName);
+        break;
+      case "fail-load":
+        console.log(`  FAIL  ${r.pkgName}  [load] ${r.detail}`);
+        failedPlugins.push(r.pkgName);
+        break;
       default:
         console.log(`  FAIL  ${r.pkgName}  ${r.error}`);
         failedPlugins.push(r.pkgName);
     }
   }
 
-  const counts = { pass: 0, warn: 0, skip: 0, fail: 0 };
-  for (const r of results) counts[r.status]++;
+  const be = { pass: 0, warn: 0, skip: 0, fail: 0 };
+  const fe = { pass: 0, fail: 0 };
+  for (const r of results) {
+    const isFe = isFrontendRole(r.role);
+    if (r.status === "fail-bundle" || r.status === "fail-load") {
+      if (isFe) fe.fail++;
+      else be.fail++;
+    } else if (isFe) {
+      fe[r.status] = (fe[r.status] ?? 0) + 1;
+    } else {
+      be[r.status] = (be[r.status] ?? 0) + 1;
+    }
+  }
+  const total = results.length;
+  const totalFail = be.fail + fe.fail;
   console.log(
-    `\n  Total: ${results.length}  Pass: ${counts.pass}  Warn: ${counts.warn}  Skip: ${counts.skip}  Fail: ${counts.fail}\n`,
+    `\n  Total: ${total}  Backend: ${be.pass} pass / ${be.warn} warn / ${be.fail} fail / ${be.skip} skip  Frontend: ${fe.pass} pass / ${fe.fail} fail\n`,
   );
 
-  const success = counts.fail === 0;
+  const success = totalFail === 0;
   writeFileSync(
     resultsFile,
     JSON.stringify({ success, failedPlugins, results }, null, 2),
   );
   return success;
 }
 
+// ---------------------------------------------------------------------------
+// Result merging
+// ---------------------------------------------------------------------------
+
+function mergeFrontendResults(bundleResults, loadResults) {
+  const loadMap = new Map();
+  for (const r of loadResults) {
+    if (!loadMap.has(r.pkgName)) loadMap.set(r.pkgName, r);
+  }
+
+  return bundleResults.map((br) => {
+    if (br.status === "fail-bundle") return br;
+
+    const lr = loadMap.get(br.pkgName);
+    if (!lr) {
+      return {
+        ...br,
+        status: "fail-load",
+        detail: "missing load probe result",
+      };
+    }
+    if (lr.status === "fail-load") return lr;
+    return { ...br, status: "pass" };
+  });
+}
+
 // ---------------------------------------------------------------------------
 // Main
 // ---------------------------------------------------------------------------
@@ -385,6 +625,15 @@ async function main() {
   console.log("\n2. Downloading OCI plugin images");
   await downloadPlugins(plugins, args.pluginsRoot);
 
+  console.log("\n2b. Validating frontend bundles");
+  const bundleResults = validateFrontendBundles(plugins, args.pluginsRoot);
+  const bundleFailCount = bundleResults.filter(
+    (r) => r.status === "fail-bundle",
+  ).length;
+  console.log(
+    `  ${bundleResults.length} frontend plugin(s) checked, ${bundleFailCount} failed`,
+  );
+
   console.log("\n3. Generating merged plugin config");
   const generatedCfg = generateMergedConfig(plugins, args.pluginsRoot);
 
@@ -397,13 +646,27 @@ async function main() {
     console.log("\n5. Waiting for readiness");
     await waitForReady(args.port, args.timeout);
 
-    console.log("\n6. Probing plugin routes");
-    const results = await probePluginRoutes(
+    console.log("\n6a. Probing backend plugin routes");
+    const backendResults = await probePluginRoutes(
+      plugins,
+      args.port,
+      args.pluginsRoot,
+    );
+
+    console.log("\n6b. Probing frontend loaded plugins");
+    const frontendLoadResults = await probeFrontendPlugins(
       plugins,
       args.port,
       args.pluginsRoot,
     );
-    success = reportAndWrite(results, args.resultsFile);
+
+    const frontendResults = mergeFrontendResults(
+      bundleResults,
+      frontendLoadResults,
+    );
+
+    const allResults = [...backendResults, ...frontendResults];
+    success = reportAndWrite(allResults, args.resultsFile);
   } finally {
     console.log("Shutting down backend...");
     await backend.stop();