ci: Change host tests to run in environment mode

.github/workflows/test-web-assets.yaml

@@ -12,6 +12,18 @@ on:
         required: false
         type: boolean
         default: false
+      boxel_environment:
+        description: |
+          BOXEL_ENVIRONMENT slug to bake into the host dist. Empty for
+          a standard-mode build (URLs like https://localhost:4201);
+          a value like "ci" produces an env-mode build whose realm URLs
+          target the per-environment Traefik hostnames
+          (https://realm-server.ci.localhost, etc.). The slug is folded
+          into the cache key and artifact name so standard-mode and
+          env-mode builds don't collide.
+        required: false
+        type: string
+        default: ""
     outputs:
       artifact_name:
         description: Artifact name containing built test web assets
@@ -46,8 +58,13 @@ jobs:
         run: |
           ref_slug="$(echo "${GITHUB_REF}" | tr '/' '_')"
           skip_catalog="${{ inputs.skip_catalog }}"
-          cache_key="test-web-assets-v2-${RUNNER_OS}-${ref_slug}-${GITHUB_SHA}-skip-catalog-${skip_catalog}"
-          artifact_name="test-web-assets-${GITHUB_SHA}-skip-catalog-${skip_catalog}"
+          boxel_env="${{ inputs.boxel_environment }}"
+          # An empty boxel_environment encodes the standard-mode build;
+          # any non-empty slug produces a distinct artifact + cache entry
+          # so env-mode and standard-mode builds for the same SHA coexist.
+          env_suffix="${boxel_env:-std}"
+          cache_key="test-web-assets-v2-${RUNNER_OS}-${ref_slug}-${GITHUB_SHA}-skip-catalog-${skip_catalog}-env-${env_suffix}"
+          artifact_name="test-web-assets-${GITHUB_SHA}-skip-catalog-${skip_catalog}-env-${env_suffix}"
           echo "cache_key=$cache_key" >> "$GITHUB_OUTPUT"
           echo "artifact_name=$artifact_name" >> "$GITHUB_OUTPUT"
 
@@ -78,6 +95,10 @@ jobs:
         working-directory: packages/host
         env:
           SKIP_CATALOG: ${{ inputs.skip_catalog }}
+          # When set, environment.js bakes the env-mode realm/matrix/icons
+          # URLs (https://realm-server.<slug>.localhost, etc.) into the
+          # vite chunks. Empty preserves the standard-mode defaults.
+          BOXEL_ENVIRONMENT: ${{ inputs.boxel_environment }}
 
       - name: Write assets manifest
         shell: bash
@@ -94,6 +115,7 @@ jobs:
             cacheKey: process.env.CACHE_KEY,
             cacheHit: process.env.CACHE_HIT === "true",
             skipCatalog: process.env.SKIP_CATALOG === "true",
+            boxelEnvironment: process.env.BOXEL_ENVIRONMENT || null,
             builtAt: new Date().toISOString(),
           };
           fs.writeFileSync(".ci/test-web-assets/manifest.json", JSON.stringify(manifest, null, 2));
@@ -102,6 +124,7 @@ jobs:
           CACHE_KEY: ${{ steps.keys.outputs.cache_key }}
           CACHE_HIT: ${{ steps.restore.outputs.cache-hit }}
           SKIP_CATALOG: ${{ inputs.skip_catalog }}
+          BOXEL_ENVIRONMENT: ${{ inputs.boxel_environment }}
 
       - name: Save cached assets
         if: steps.restore.outputs.cache-hit != 'true'

mise-tasks/services/test-realms

@@ -9,6 +9,17 @@ SCRIPTS_DIR="./scripts"
 # server never writes into the repo's source tree.
 TEST_REALM_DIR=$(mktemp -d)
 cp -R ../test-realm-cards/contents/. "$TEST_REALM_DIR"/
+# Diagnostics: env-mode CI has been seeing the test realm-server's
+# from-scratch index finish with `files_completed=0` for both
+# `/test/` (this temp dir) and `/node-test/` (./tests/fixtures/realistic),
+# which means the realm-server's `_mtimes` endpoint is walking what
+# looks like an empty directory. Log file counts at this point so the
+# CI log makes it obvious whether the cp succeeded, whether the
+# checked-in fixtures directory is populated, and whether anything
+# wipes them between here and the realm-server boot.
+echo "[test-realms diag] TEST_REALM_DIR=$TEST_REALM_DIR file_count=$(ls -1A "$TEST_REALM_DIR" 2>/dev/null | wc -l | tr -d ' ')"
+echo "[test-realms diag] ../test-realm-cards/contents file_count=$(ls -1A ../test-realm-cards/contents 2>/dev/null | wc -l | tr -d ' ')"
+echo "[test-realms diag] ./tests/fixtures/realistic file_count=$(ls -1A ./tests/fixtures/realistic 2>/dev/null | wc -l | tr -d ' ')"
 
 # In environment mode, share the dev icons server; otherwise start our own
 if [ -z "$BOXEL_ENVIRONMENT" ]; then

packages/host/app/config/environment.ts

@@ -39,6 +39,7 @@ export default config as {
   resolvedCatalogRealmURL: string | undefined;
   resolvedSkillsRealmURL: string;
   resolvedOpenRouterRealmURL: string | undefined;
+  resolvedTestRealmURL: string;
   hostsOwnAssets: boolean;
   realmsServed?: string[];
   logLevels: string;

packages/host/app/services/network.ts

@@ -1,6 +1,8 @@
 import type Owner from '@ember/owner';
 import Service, { service } from '@ember/service';
 
+import { isTesting } from '@embroider/macros';
+
 import {
   VirtualNetwork,
   authorizationMiddleware,
@@ -83,6 +85,25 @@ export default class NetworkService extends Service {
         config.resolvedOpenRouterRealmURL,
       );
     }
+    // Some test fixture content (JSON card files under tests/cards/, embedded
+    // card ids in test data) refers to the live test realm by its standard-
+    // mode URL `https://localhost:4202/test/`. In environment mode the live
+    // test realm is served at a per-environment Traefik hostname. Mapping
+    // the standard-mode URL onto whatever the running test realm-server
+    // serves lets the same fixture content resolve under either mode.
+    // Gated on isTesting() so the mapping never reaches prod fetches.
+    if (isTesting()) {
+      let hardcodedTestRealmURL = new URL('https://localhost:4202/test/');
+      let resolvedTestRealmURL = new URL(
+        withTrailingSlash(config.resolvedTestRealmURL),
+      );
+      if (resolvedTestRealmURL.href !== hardcodedTestRealmURL.href) {
+        virtualNetwork.addURLMapping(
+          hardcodedTestRealmURL,
+          resolvedTestRealmURL,
+        );
+      }
+    }
     return virtualNetwork;
   }
 

packages/host/config/environment.js

@@ -67,6 +67,7 @@ function environmentDefaults() {
       catalogRealmURL: 'https://localhost:4201/catalog/',
       skillsRealmURL: 'https://localhost:4201/skills/',
       openRouterRealmURL: 'https://localhost:4201/openrouter/',
+      testRealmURL: 'https://localhost:4202/test/',
     };
   }
   let slug = getEnvSlug();
@@ -87,6 +88,10 @@ function environmentDefaults() {
     catalogRealmURL: `https://${realmHost}/catalog/`,
     skillsRealmURL: `https://${realmHost}/skills/`,
     openRouterRealmURL: `https://${realmHost}/openrouter/`,
+    // mise-tasks/services/test-realms registers the live test realm at
+    // `https://realm-test.${slug}.localhost/test/` in env mode (the
+    // counterpart to standard mode's `https://localhost:4202/test/`).
+    testRealmURL: `https://realm-test.${slug}.localhost/test/`,
   };
 }
 
@@ -165,6 +170,24 @@ module.exports = function (environment) {
       process.env.RESOLVED_SKILLS_REALM_URL || defaults.skillsRealmURL,
     resolvedOpenRouterRealmURL:
       process.env.RESOLVED_OPENROUTER_REALM_URL || defaults.openRouterRealmURL,
+    // The live test realm-server's /test/ realm — used by host tests
+    // that load source modules from it via
+    // `tests/helpers#testModuleRealm`. Derived from BOXEL_ENVIRONMENT via
+    // `defaults.testRealmURL` above (localhost:4202 in standard mode,
+    // realm-test.<slug>.localhost in env mode). Explicit
+    // `REALM_TEST_URL` overrides take precedence for non-CI consumers
+    // that want a custom test realm endpoint. The override accepts
+    // either a base host URL (which gets `/test/` appended) or a value
+    // that already names the `/test` realm — without the latter case
+    // a path like `https://my-host/test/` would become
+    // `https://my-host/test/test/`.
+    resolvedTestRealmURL: (() => {
+      if (!process.env.REALM_TEST_URL) return defaults.testRealmURL;
+      let normalized = process.env.REALM_TEST_URL.replace(/\/$/, '');
+      return normalized.endsWith('/test')
+        ? `${normalized}/`
+        : `${normalized}/test/`;
+    })(),
     featureFlags: {},
   };
 

packages/host/package.json

@@ -24,7 +24,7 @@
     "start": "node scripts/vite-serve.js",
     "serve:dist": "node scripts/serve-dist.js",
     "test": "concurrently \"pnpm:lint\" \"pnpm:test:*\" --names \"lint,test:\"",
-    "test-with-percy": "percy exec --parallel -- pnpm test:wait-for-servers",
+    "test-with-percy": "percy exec --parallel -- pnpm ember-test-pre-built",
     "test:wait-for-servers": "./scripts/test-wait-for-servers.sh",
     "test:live": "./scripts/live-test-wait-for-servers.sh",
     "ember-test-pre-built": "sleep 15 && ember exam --path ./dist --split $HOST_TEST_PARTITION_COUNT --partition $HOST_TEST_PARTITION --preserve-test-name",

packages/host/scripts/live-test-wait-for-servers.sh

@@ -1,8 +1,22 @@
 #! /bin/sh
 
 READY_PATH="_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson"
-BASE_REALM_READY="https-get://localhost:4201/base/${READY_PATH}"
-SYNAPSE_URL="http://localhost:8008"
+
+# Default to the standard-mode ports; env mode (BOXEL_ENVIRONMENT set)
+# exports REALM_BASE_URL=https://realm-server.<slug>.localhost and
+# MATRIX_URL_VAL=https://matrix.<slug>.localhost via env-vars.sh, so
+# fall back to those when present.
+REALM_BASE_URL="${REALM_BASE_URL:-https://localhost:4201}"
+MATRIX_URL_VAL="${MATRIX_URL_VAL:-http://localhost:8008}"
+realm_host="${REALM_BASE_URL#http://}"
+realm_host="${realm_host#https://}"
+realm_host="${realm_host%/}"
+case "$REALM_BASE_URL" in
+  https://*) realm_scheme="https-get" ;;
+  *)         realm_scheme="http-get"  ;;
+esac
+BASE_REALM_READY="${realm_scheme}://${realm_host}/base/${READY_PATH}"
+SYNAPSE_URL="$MATRIX_URL_VAL"
 SMTP_4_DEV_URL="http://localhost:5001"
 
 # Pick wait-on's protocol prefix from whichever scheme the caller used.
@@ -32,7 +46,7 @@ else
   # passing assert). Skills is small and is hosted by every host CI job
   # already, so waiting on its readiness adds little beyond what base
   # already incurs.
-  SKILLS_REALM_READY="https-get://localhost:4201/skills/${READY_PATH}"
+  SKILLS_REALM_READY="${realm_scheme}://${realm_host}/skills/${READY_PATH}"
   READY_URLS="$BASE_REALM_READY|$SKILLS_REALM_READY|$SYNAPSE_URL|$SMTP_4_DEV_URL"
 fi
 

packages/host/tests/acceptance/code-submode-test.ts

@@ -44,6 +44,7 @@ import {
   SYSTEM_CARD_FIXTURE_CONTENTS,
   setMonacoContent,
   setupLocalIndexing,
+  testModuleRealm,
   testRealmURL,
   visitOperatorMode,
   setupAuthEndpoints,
@@ -1273,12 +1274,12 @@ module('Acceptance | code submode tests', function (_hooks) {
     });
     module('with connection to test realm', function (hooks) {
       hooks.beforeEach(function () {
-        setActiveRealms([testRealmURL, 'https://localhost:4202/test/']);
+        setActiveRealms([testRealmURL, `${testModuleRealm}`]);
       });
       test('code submode handles binary files', async function (assert) {
         await visitOperatorMode({
           submode: 'code',
-          codePath: `https://localhost:4202/test/mango.png`,
+          codePath: `${testModuleRealm}mango.png`,
         });
 
         await waitFor('[data-test-binary-info]');

packages/host/tests/acceptance/code-submode/file-tree-test.ts

@@ -22,6 +22,7 @@ import {
   elementIsVisible,
   setupLocalIndexing,
   setupRealmCacheTeardown,
+  testModuleRealm,
   testRealmURL,
   setupAcceptanceTestRealm,
   SYSTEM_CARD_FIXTURE_CONTENTS,
@@ -478,7 +479,7 @@ module('Acceptance | code submode | file-tree tests', function (hooks) {
     // go to a file with different realm
     await fillIn(
       '[data-test-card-url-bar-input]',
-      `https://localhost:4202/test/mango.png`,
+      `${testModuleRealm}mango.png`,
     );
     await triggerKeyEvent(
       '[data-test-card-url-bar-input]',
@@ -522,7 +523,7 @@ module('Acceptance | code submode | file-tree tests', function (hooks) {
 
     await fillIn(
       '[data-test-card-url-bar-input]',
-      `https://localhost:4202/test/mango.png`,
+      `${testModuleRealm}mango.png`,
     );
     await triggerKeyEvent(
       '[data-test-card-url-bar-input]',
@@ -824,7 +825,7 @@ module('Acceptance | code submode | file-tree tests', function (hooks) {
 
     await fillIn(
       '[data-test-card-url-bar-input]',
-      `https://localhost:4202/test/mango.png`,
+      `${testModuleRealm}mango.png`,
     );
     await triggerKeyEvent(
       '[data-test-card-url-bar-input]',

packages/host/tests/acceptance/code-submode/recent-files-test.ts

@@ -21,6 +21,7 @@ import {
   percySnapshot,
   setupLocalIndexing,
   setupRealmCacheTeardown,
+  testModuleRealm,
   testRealmURL,
   setupAcceptanceTestRealm,
   SYSTEM_CARD_FIXTURE_CONTENTS,
@@ -318,7 +319,7 @@ module('Acceptance | code submode | recent files tests', function (hooks) {
   test('recent file links are shown', async function (assert) {
     setRecentFiles([
       [testRealmURL, 'index.json'],
-      ['https://localhost:4202/test/', 'français.json'],
+      [`${testModuleRealm}`, 'français.json'],
       // @ts-ignore error on purpose
       'a-non-url-to-ignore',
     ]);
@@ -404,7 +405,7 @@ module('Acceptance | code submode | recent files tests', function (hooks) {
       [testRealmURL, 'index.json', null],
       [testRealmURL, 'français.json', null],
       [testRealmURL, 'Person/1.json', null],
-      ['https://localhost:4202/test/', 'français.json', null],
+      [`${testModuleRealm}`, 'français.json', null],
     ]);
   });
 

packages/host/tests/acceptance/commands-test.gts

@@ -53,6 +53,7 @@ import {
   addSkillToAiAssistant,
   setupLocalIndexing,
   setupOnSave,
+  testModuleRealm,
   testRealmURL,
   testRRI,
   setupAcceptanceTestRealm,
@@ -399,7 +400,7 @@ module('Acceptance | Commands tests', function (hooks) {
             },
             meta: {
               adoptsFrom: {
-                module: 'https://localhost:4202/test/ai-command-example',
+                module: `${testModuleRealm}ai-command-example`,
                 name: 'AiCommandExample',
               },
             },