Replace Docker-based smoke tests with native Node.js harness

Boot a minimal Backstage backend directly on the runner using
createBackend() + dynamicPluginsFeatureLoader, probe /api/<pluginId>
routes, and report results as structured JSON. Includes core bundled
plugins (catalog, auth, permission, scaffolder, events, search, proxy)
so dynamic plugins resolve their dependencies correctly.

Made-with: Cursor

Author: rostalan

.github/workflows/run-workspace-smoke-tests.yaml

@@ -15,13 +15,13 @@ on:
         description: Newline-separated list of plugins that failed to load
         value: ${{ jobs.run.outputs.failed-plugins }}
       error-logs:
-        description: Extracted error messages from container logs
+        description: Extracted error messages from smoke test output
         value: ${{ jobs.run.outputs.error-logs }}
 
 jobs:
   run:
     runs-on: ubuntu-latest
-    timeout-minutes: 25
+    timeout-minutes: 15
     permissions:
       contents: read
       packages: read
@@ -36,139 +36,81 @@ jobs:
           name: smoke-test-artifacts
           path: ./artifacts
 
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
         with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          node-version: "20"
 
-      - name: Start RHDH with test plugins config
+      - name: Install skopeo
         run: |
-          set -euo pipefail
-          ls -la ./artifacts/ || true
-          
+          if ! command -v skopeo &>/dev/null; then
+            sudo apt-get update -qq
+            sudo apt-get install -y -qq skopeo
+          fi
+          skopeo --version
+
+      - name: Authenticate to GHCR
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | skopeo login ghcr.io -u "${{ github.actor }}" --password-stdin
+
+      - name: Verify artifacts
+        run: |
+          ls -la ./artifacts/
+          ls -la ./artifacts/harness/ || true
           if [ ! -f "./artifacts/dynamic-plugins.test.yaml" ]; then
-            echo "Error: dynamic-plugins.test.yaml not found in artifacts"
+            echo "Error: dynamic-plugins.test.yaml not found"
             exit 1
           fi
+          echo "=== dynamic-plugins.test.yaml (first 40 lines) ==="
+          head -40 ./artifacts/dynamic-plugins.test.yaml || true
 
-          echo "dynamic-plugins.test.yaml contents:"
-          sed -n '1,40p' ./artifacts/dynamic-plugins.test.yaml || true
-          
-          # Build Docker run command with conditional volume mounts
-          ENV_ARGS=$( [ -f ./artifacts/test.env ] && echo "--env-file ./artifacts/test.env" || echo "" )
-          DOCKER_CMD="docker run -d --name rhdh -p 7007:7007 $ENV_ARGS"
-          if [ -f "./artifacts/app-config.yaml" ]; then
-            DOCKER_CMD="$DOCKER_CMD -v "$(pwd)"/artifacts/app-config.yaml:/opt/app-root/src/app-config.yaml"
-          fi
-          if [ -f "./artifacts/app-config.test.yaml" ]; then
-            DOCKER_CMD="$DOCKER_CMD -v "$(pwd)"/artifacts/app-config.test.yaml:/opt/app-root/src/app-config.test.yaml"
-          fi
-          DOCKER_CMD="$DOCKER_CMD -v "$(pwd)"/artifacts/dynamic-plugins.test.yaml:/opt/app-root/src/dynamic-plugins.yaml"
-          
-          # Add Docker config and environment variables
-          echo "Using docker auth file: $HOME/.docker/config.json"
-          DOCKER_CMD="$DOCKER_CMD -v $HOME/.docker/config.json:/root/.docker/config.json:ro"
-          DOCKER_CMD="$DOCKER_CMD -e REGISTRY_AUTH_FILE=/root/.docker/config.json"
-          
-          # Derive image tag from target branch
-          TARGET_BRANCH="${{ inputs.target-branch }}"
-          if [[ "$TARGET_BRANCH" =~ ^release-([0-9]+\.[0-9]+)$ ]]; then
-            IMAGE_TAG="next-${BASH_REMATCH[1]}"
-          else
-            IMAGE_TAG="next"
-          fi
-          echo "Using RHDH image tag: $IMAGE_TAG (target branch: $TARGET_BRANCH)"
-          
-          # Add image and command
-          DOCKER_CMD="$DOCKER_CMD --entrypoint /bin/bash quay.io/rhdh-community/rhdh:${IMAGE_TAG} -c '"
-          DOCKER_CMD="$DOCKER_CMD set -ex; "
-          DOCKER_CMD="$DOCKER_CMD PLUGINS_ROOT=/opt/app-root/src/dynamic-plugins-root; "
-          DOCKER_CMD="$DOCKER_CMD GENERATED_CONFIG=\$PLUGINS_ROOT/app-config.dynamic-plugins.yaml; "
-          DOCKER_CMD="$DOCKER_CMD INSTALL_SCRIPT=/opt/app-root/src/install-dynamic-plugins.sh; "
-          DOCKER_CMD="$DOCKER_CMD mkdir -p \$PLUGINS_ROOT; "
-          DOCKER_CMD="$DOCKER_CMD \$INSTALL_SCRIPT \$PLUGINS_ROOT; "
-          DOCKER_CMD="$DOCKER_CMD exec node packages/backend"
-          
-          # Add config files to command (optional)
-          [ -f "./artifacts/app-config.yaml" ] && DOCKER_CMD="$DOCKER_CMD --config /opt/app-root/src/app-config.yaml"
-          [ -f "./artifacts/app-config.test.yaml" ] && DOCKER_CMD="$DOCKER_CMD --config /opt/app-root/src/app-config.test.yaml"
-          DOCKER_CMD="$DOCKER_CMD --config /opt/app-root/src/dynamic-plugins.yaml"
-          DOCKER_CMD="$DOCKER_CMD --config \$GENERATED_CONFIG'"
-          
-          echo "Running: $DOCKER_CMD"
-          eval "$DOCKER_CMD"
-
-      - name: Wait for RHDH to be ready
+      - name: Install smoke test dependencies
+        working-directory: ./artifacts/harness
+        run: npm install --ignore-scripts 2>&1 | tail -5
+
+      - name: Run smoke test
+        id: smoke-test
+        working-directory: ./artifacts/harness
         run: |
-          set -e
-          for i in $(seq 1 10); do
-            if curl -fsS http://localhost:7007/health >/dev/null; then
-              echo "RHDH is ready"; exit 0; fi
-            echo "Waiting for RHDH... (Attempt ${i}/10)"
-            # Check if container is still running
-            if ! docker ps | grep -q rhdh; then
-              echo "Container stopped unexpectedly."
-              exit 1
-            fi
-            sleep 10
-          done
-          echo "RHDH did not become ready in time."
-          exit 1
+          set -o pipefail
 
-      - name: List installed plugins
-        run: docker exec rhdh ls -l /opt/app-root/src/dynamic-plugins-root
+          CONFIG_ARGS="--config app-config.yaml"
+          [ -f ../app-config.yaml ] && CONFIG_ARGS="$CONFIG_ARGS --config ../app-config.yaml"
+          [ -f ../app-config.test.yaml ] && CONFIG_ARGS="$CONFIG_ARGS --config ../app-config.test.yaml"
 
-      - name: Print generated dynamic plugins config
-        run: docker exec rhdh cat /opt/app-root/src/dynamic-plugins-root/app-config.dynamic-plugins.yaml
+          ENV_ARGS=""
+          [ -f ../test.env ] && ENV_ARGS="--env-file ../test.env"
 
-      - name: Verify plugin loading
+          echo "Running: node smoke-test.mjs --plugins-yaml ../dynamic-plugins.test.yaml $CONFIG_ARGS $ENV_ARGS"
+
+          node smoke-test.mjs \
+            --plugins-yaml ../dynamic-plugins.test.yaml \
+            $CONFIG_ARGS \
+            $ENV_ARGS \
+            2>&1 | tee ../smoke-test.log
+
+      - name: Collect results
         id: collect-results
+        if: always()
         run: |
-          set -e
-          PLUGINS=$(grep -Eo '!([^[:space:]]+)' ./artifacts/dynamic-plugins.test.yaml | sed -e 's/^!//' -e 's/"$//' | sort -u)
-          if [ -z "$PLUGINS" ]; then
-            echo "No plugins found in dynamic-plugins.test.yaml"
-            echo "success=false" >> "$GITHUB_OUTPUT"
-            echo "failed-plugins<<EOF" >> "$GITHUB_OUTPUT"
-            echo "(no-plugins)" >> "$GITHUB_OUTPUT"
-            echo "EOF" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          LOGS=$(docker logs rhdh || true)
-          failures=()
-          echo "===== Checking logs for plugin loaded messages"
-          for plugin in $PLUGINS; do
-            echo "Asserting plugin loaded: $plugin"
-            # Match either the unpack path or the canonical "loaded dynamic ... plugin '<pkg>' from" message
-            if echo "$LOGS" | grep -qiE "loaded dynamic .* plugin .*${plugin}(-dynamic)?'" ; then
-              echo "Plugin loaded: $plugin"
-            else
-              echo "Plugin NOT loaded: $plugin"
-              failures+=("$plugin")
-            fi
-          done
-          if echo "$LOGS" | grep -E "(InstallException|Error while adding OCI plugin|Failed to load dynamic plugin|dynamic plugin.*error)" >/dev/null; then
-            echo "Detected dynamic plugin loading errors in logs"
-            failures+=("(log-errors)")
-          fi
-          if [ ${#failures[@]} -eq 0 ]; then
-            echo "success=true" >> "$GITHUB_OUTPUT"
+          RESULTS_FILE="./artifacts/harness/results.json"
+          if [ -f "$RESULTS_FILE" ]; then
+            SUCCESS=$(jq -r '.success' "$RESULTS_FILE")
+            FAILED=$(jq -r '.failedPlugins | join("\n")' "$RESULTS_FILE")
+            echo "success=$SUCCESS" >> "$GITHUB_OUTPUT"
             echo "failed-plugins<<EOF" >> "$GITHUB_OUTPUT"
-            echo "" >> "$GITHUB_OUTPUT"
+            echo "$FAILED" >> "$GITHUB_OUTPUT"
             echo "EOF" >> "$GITHUB_OUTPUT"
           else
             echo "success=false" >> "$GITHUB_OUTPUT"
             echo "failed-plugins<<EOF" >> "$GITHUB_OUTPUT"
-            printf "%s\n" "${failures[@]}" >> "$GITHUB_OUTPUT"
+            echo "(results-file-missing)" >> "$GITHUB_OUTPUT"
             echo "EOF" >> "$GITHUB_OUTPUT"
           fi
 
-      - name: Fail if any plugin failed to load
+      - name: Fail if smoke test failed
         if: ${{ steps.collect-results.outputs.success != 'true' }}
         run: |
-          echo "The following plugins failed to load to RHDH:"
+          echo "Smoke test failed. Failed plugins:"
           echo "${{ steps.collect-results.outputs.failed-plugins }}"
           exit 1
 
@@ -177,30 +119,21 @@ jobs:
         id: capture-errors
         continue-on-error: true
         run: |
-          if ! docker ps -a | grep -q rhdh; then
-            echo "error-logs<<EOF" >> "$GITHUB_OUTPUT"
-            echo "'rhdh' container was not available. Check earlier steps for startup errors." >> "$GITHUB_OUTPUT"
-            echo "EOF" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          
-          LOGS=$(docker logs rhdh 2>&1 || echo "Failed to retrieve logs")
-          
-          # Extract errors from logs with short context 
-          ERROR_LINES=$(echo "$LOGS" | grep -iE -B 2 -A 2 "(error|exception|failed|failure|installException)" | grep -vE "(no errors|successfully|resolved|fixed)" | grep -v "^--$" | tail -n 100 || echo "")
-          
+          LOG_FILE="./artifacts/smoke-test.log"
           echo "error-logs<<EOF" >> "$GITHUB_OUTPUT"
-          if [ -n "$ERROR_LINES" ] && [ "$ERROR_LINES" != "" ]; then
-            echo "$ERROR_LINES" >> "$GITHUB_OUTPUT"
+          if [ -f "$LOG_FILE" ]; then
+            grep -iE -B 1 -A 1 "(error|exception|failed|failure)" "$LOG_FILE" \
+              | grep -vE "(no errors|successfully|resolved)" \
+              | tail -100 || echo "No error patterns found in logs."
           else
-            echo "No specific error patterns found in container logs. Check full workflow logs for details." >> "$GITHUB_OUTPUT"
+            echo "No smoke test log available."
           fi
           echo "EOF" >> "$GITHUB_OUTPUT"
 
-      - name: Print container logs
+      - name: Print full smoke test log
         if: always()
-        run: docker logs rhdh || true
+        run: cat ./artifacts/smoke-test.log 2>/dev/null || echo "No log file"
 
       - name: Cleanup
         if: always()
-        run: docker rm -f rhdh || true
+        run: rm -rf ./artifacts/harness/dynamic-plugins-root ./artifacts/harness/.tmp-oci || true

.github/workflows/workspace-tests.yaml

@@ -119,7 +119,7 @@ jobs:
               console.log('Missing PR or commit; skipping pending status');
               return;
             }
-            
+
             await github.rest.repos.createCommitStatus({
               owner: context.repo.owner,
               repo: context.repo.repo,
@@ -205,7 +205,7 @@ jobs:
           # Always include the root-level default config
           ROOT_CONFIG="$GITHUB_WORKSPACE/smoke-tests/app-config.yaml"
           [ -f "$ROOT_CONFIG" ] && cp "$ROOT_CONFIG" "$OUT_DIR/app-config.yaml"
-          
+
           # Read workspace-wide test.env if it exists
           WORKSPACE_ENV_FILE="$WORKSPACE_PATH/smoke-tests/test.env"
           WORKSPACE_ENV_CONTENT=""
@@ -309,6 +309,16 @@ jobs:
           echo "plugins-metadata-complete=$PLUGINS_METADATA_COMPLETE" >> "$GITHUB_OUTPUT"
           echo "skip-tests-missing-env=$SKIP_TESTS_MISSING_ENV" >> "$GITHUB_OUTPUT"
 
+      - name: Stage smoke test harness
+        env:
+          WORKSPACE_PATH: ${{ needs.resolve.outputs.workspace }}
+        run: |
+          HARNESS_DIR="$WORKSPACE_PATH/smoke-tests/harness"
+          mkdir -p "$HARNESS_DIR"
+          cp smoke-tests/package.json "$HARNESS_DIR/"
+          cp smoke-tests/smoke-test.mjs "$HARNESS_DIR/"
+          cp smoke-tests/app-config.yaml "$HARNESS_DIR/"
+
       - name: Upload smoke test artifacts
         uses: actions/upload-artifact@v4
         with:
@@ -352,11 +362,11 @@ jobs:
             const workspace = core.getInput('workspace');
             const pluginsMetadataComplete = core.getInput('plugins_metadata_complete') === 'true';
             const skipTestsMissingEnv = core.getInput('skip_tests_missing_env') === 'true';
-            
+
             let statusDescription = 'Skipped';
             let commentDetail = ' skipped for an unknown reason. Check workflow run for details.\n';
             let summaryDetail = 'Unknown reason. Check workflow run for details.';
-            
+
             if (!workspace || workspace === '') {
               statusDescription = 'Skipped: PR doesn\'t touch one workspace';
               commentDetail = ' skipped: PR doesn\'t touch exactly one workspace.\n';
@@ -370,7 +380,7 @@ jobs:
               commentDetail = ' skipped: missing plugin metadata files (`<workspace>/metadata/*.yaml`).\n';
               summaryDetail = 'Missing plugin metadata files (`<workspace>/metadata/*.yaml`).';
             }
-            
+
             if (overlayCommit) {
               await github.rest.repos.createCommitStatus({
                 owner: context.repo.owner,
@@ -383,7 +393,7 @@ jobs:
               });
               console.log(`Set success status on ${overlayCommit} (skipped for valid reason)`);
             }
-            
+
             if (pr) {
               await github.rest.issues.createComment({
                 owner: context.repo.owner,
@@ -392,7 +402,7 @@ jobs:
                 body: `:warning: \n[Smoke test workflow](${runUrl})${commentDetail}`,
               });
             }
-            
+
             await core.summary
               .addRaw('\n### Tests Skipped\n\n' + summaryDetail)
               .write();
@@ -431,10 +441,10 @@ jobs:
             const errorLogs = (process.env.ERROR_LOGS || '').trim();
             const pr = Number(process.env.PR_NUMBER);
             const overlayCommit = process.env.OVERLAY_COMMIT;
-            
+
             const success = smokeTestsResult === 'success' && successOutput === 'true';
             let failureReason = '';
-            
+
             if (!success) {
               switch (smokeTestsResult) {
                 case 'failure':
@@ -450,7 +460,7 @@ jobs:
                   failureReason = `Smoke tests ended in an unexpected state: ${smokeTestsResult}. Check the workflow logs for details.`;
               }
             }
-            
+
             // Write step summary (always, even if PR is unavailable)
             let summary;
             if (success) {
@@ -465,23 +475,23 @@ jobs:
               }
             }
             await core.summary.addRaw(summary).write();
-            
+
             if (!pr) {
               console.log('No PR associated; skipping status and comment');
               return;
             }
-            
+
             // Get current PR head SHA
             const { data: prData } = await github.rest.pulls.get({
               owner: context.repo.owner,
               repo: context.repo.repo,
               pull_number: pr,
             });
-            
+
             // Use PR head if different from overlayCommit (/smoketest command), else use overlayCommit (immediate publish)
             const sha = prData.head.sha !== overlayCommit ? prData.head.sha : overlayCommit;
             console.log(`Status SHA: ${sha} (PR head: ${prData.head.sha}, overlay: ${overlayCommit})`);
-            
+
             await github.rest.repos.createCommitStatus({
               owner: context.repo.owner,
               repo: context.repo.repo,
@@ -491,7 +501,7 @@ jobs:
               target_url: runUrl,
               context: 'smoketest',
             });
-            
+
             let body;
             if (success) {
               body = `:white_check_mark: [Smoke tests workflow](${runUrl}) passed. All plugins loaded successfully.\n`;
@@ -504,7 +514,7 @@ jobs:
                 body += `\n\n<details><summary>Error logs from container</summary>\n\n\`\`\`\n${errorLogs}\n\`\`\`\n\n</details>`;
               }
             }
-            
+
             await github.rest.issues.createComment({
               issue_number: pr,
               owner: context.repo.owner,