fix: Restore kernel-browser-runtime sourcemaps in extension (#575)

When we introduced the `kernel-browser-runtime` package, we effectively
broke debugging the kernel by breaking its sourcemaps. This happened
because we copied its build output (also from Vite) using
`vite-plugin-static-copy`. To fix this, this PR modifies the extension
`vite.config.ts` to build `kernel-browser-runtime` directly.

A bigger mystery is why Vite doesn't pull in the sourcemaps of bundled
dependencies that ship them. The symptom of this is that we see
transpiled `.js` and `.mjs` files in the debugger rather than original
sources. However, this is still a significant improvement over the
status quo, and we can address the larger sourcemap problem later.

Author: rekmarks

packages/extension/scripts/build-constants.mjs

@@ -2,8 +2,15 @@
 
 import path from 'path';
 
-export const sourceDir = './src';
-export const buildDir = path.resolve(sourceDir, '../dist');
+const dirname = path.dirname(new URL(import.meta.url).pathname);
+export const rootDir = path.resolve(dirname, '../..');
+const extensionDir = path.resolve(rootDir, 'extension');
+export const sourceDir = path.resolve(extensionDir, 'src');
+export const outDir = path.resolve(extensionDir, 'dist');
+export const kernelBrowserRuntimeSrcDir = path.resolve(
+  rootDir,
+  'kernel-browser-runtime/src',
+);
 
 /**
  * @type {import('@ocap/vite-plugins').PreludeRecord}
@@ -12,4 +19,5 @@ export const trustedPreludes = {
   background: {
     path: path.resolve(sourceDir, 'env/background-trusted-prelude.js'),
   },
+  'kernel-worker': { content: "import './endoify.js';" },
 };

packages/extension/src/manifest.json

@@ -12,7 +12,7 @@
   },
   "permissions": ["offscreen", "unlimitedStorage"],
   "sandbox": {
-    "pages": ["browser-runtime/vat/iframe.html"]
+    "pages": ["iframe.html"]
   },
   "content_security_policy": {
     "extension_pages": "script-src 'self' 'wasm-unsafe-eval'; object-src 'none'; frame-ancestors 'none';",

packages/extension/src/offscreen.ts

@@ -51,7 +51,7 @@ async function makeKernelWorker(): Promise<{
   kernelStream: DuplexStream<JsonRpcResponse, JsonRpcCall>;
   vatWorkerService: VatWorkerServer;
 }> {
-  const worker = new Worker('browser-runtime/kernel-worker/index.js', {
+  const worker = new Worker('kernel-worker.js', {
     type: 'module',
   });
 
@@ -69,7 +69,7 @@ async function makeKernelWorker(): Promise<{
     (vatId) =>
       makeIframeVatWorker({
         id: vatId,
-        iframeUri: 'browser-runtime/vat/iframe.html',
+        iframeUri: 'iframe.html',
         getPort: initializeMessageChannel,
         logger: logger.subLogger({
           tags: ['iframe-vat-worker', vatId],

packages/extension/test/build/build-tests.mjs

@@ -2,7 +2,7 @@ import { promises as fs } from 'fs';
 import path from 'path';
 
 import {
-  buildDir,
+  outDir,
   sourceDir,
   trustedPreludes,
 } from '../../scripts/build-constants.mjs';
@@ -12,31 +12,24 @@ const { hasOwn } = Object;
 const untransformedFiles = [
   {
     sourcePath: path.resolve('../kernel-shims/dist/endoify.js'),
-    buildPath: path.resolve(buildDir, 'endoify.js'),
+    buildPath: path.resolve(outDir, 'endoify.js'),
   },
   {
     sourcePath: path.resolve(sourceDir, 'env/dev-console.js'),
-    buildPath: path.resolve(buildDir, 'dev-console.js'),
+    buildPath: path.resolve(outDir, 'dev-console.js'),
   },
   ...Object.values(trustedPreludes).map((prelude) => {
     if (hasOwn(prelude, 'path')) {
       return {
         sourcePath: prelude.path,
-        buildPath: path.join(buildDir, path.basename(prelude.path)),
+        buildPath: path.join(outDir, path.basename(prelude.path)),
       };
     }
 
-    const preludePath = /^import ["']([^"']+)["']/iu.exec(prelude.content)[1];
-    if (!preludePath) {
-      throw new Error('No prelude path found in content');
-    }
-
-    return {
-      sourcePath: preludePath,
-      buildPath: path.join(buildDir, path.basename(preludePath)),
-    };
+    // This is a "content" prelude, which does not specify an original source path.
+    return undefined;
   }),
-];
+].filter(Boolean);
 
 await runTests();
 
@@ -58,7 +51,7 @@ async function runTests() {
  * Test that shims and preludes are packaged untransformed.
  */
 async function checkUntransformed() {
-  for (const { buildPath, sourcePath } of untransformedFiles) {
+  for (const { sourcePath, buildPath } of untransformedFiles) {
     const [originalContent, builtContent] = await Promise.all([
       fs.readFile(sourcePath, 'utf8'),
       fs.readFile(buildPath, 'utf8'),
@@ -76,7 +69,7 @@ async function checkUntransformed() {
  */
 async function checkTrustedPreludes() {
   for (const [outputFileName, prelude] of Object.entries(trustedPreludes)) {
-    const outputFilePath = path.join(buildDir, `${outputFileName}.js`);
+    const outputFilePath = path.join(outDir, `${outputFileName}.js`);
     const outputFileContent = await fs.readFile(outputFilePath, 'utf8');
     const expectedImportStatement = hasOwn(prelude, 'path')
       ? `import "./${path.basename(prelude.path)}";`

packages/extension/vite.config.ts

@@ -2,6 +2,7 @@
 /// <reference types="vitest" />
 
 import {
+  deduplicateAssets,
   extensionDev,
   htmlTrustedPrelude,
   jsTrustedPrelude,
@@ -14,8 +15,10 @@ import { viteStaticCopy } from 'vite-plugin-static-copy';
 import type { Target } from 'vite-plugin-static-copy';
 
 import {
+  rootDir,
+  kernelBrowserRuntimeSrcDir,
+  outDir,
   sourceDir,
-  buildDir,
   trustedPreludes,
 } from './scripts/build-constants.mjs';
 
@@ -25,15 +28,11 @@ import {
  */
 const staticCopyTargets: readonly (string | Target)[] = [
   // The extension manifest
-  'manifest.json',
-  // External modules
-  'env/dev-console.js',
-  'env/background-trusted-prelude.js',
-  '../../kernel-shims/dist/endoify.js',
-  {
-    src: '../../kernel-browser-runtime/dist/static/*',
-    dest: './browser-runtime',
-  },
+  path.resolve(sourceDir, 'manifest.json'),
+  // Trusted prelude-related
+  path.resolve(sourceDir, 'env/dev-console.js'),
+  path.resolve(sourceDir, 'env/background-trusted-prelude.js'),
+  path.resolve(rootDir, 'kernel-shims/dist/endoify.js'),
 ];
 
 // https://vitejs.dev/config/
@@ -45,16 +44,26 @@ export default defineConfig(({ mode }) => {
   }
 
   return {
-    root: sourceDir,
+    root: rootDir,
 
     build: {
+      assetsDir: '',
       emptyOutDir: true,
-      outDir: buildDir,
+      // Disable Vite's module preload, which may cause SES-dependent code to run before lockdown.
+      modulePreload: false,
+      outDir,
       rollupOptions: {
         input: {
           background: path.resolve(sourceDir, 'background.ts'),
           offscreen: path.resolve(sourceDir, 'offscreen.html'),
           popup: path.resolve(sourceDir, 'popup.html'),
+          // kernel-browser-runtime
+          'kernel-worker': path.resolve(
+            kernelBrowserRuntimeSrcDir,
+            'kernel-worker',
+            'kernel-worker.ts',
+          ),
+          vat: path.resolve(kernelBrowserRuntimeSrcDir, 'vat', 'iframe.html'),
         },
         output: {
           entryFileNames: '[name].js',
@@ -82,10 +91,30 @@ export default defineConfig(({ mode }) => {
         silent: isDev,
       }),
       viteChecker({ typescript: { tsconfigPath: 'tsconfig.build.json' } }),
-      // Import sourcemaps from our own libraries
-      // For whatever reason, the types don't match, but it works
+      // Deduplicate sqlite-wasm assets
+      deduplicateAssets({
+        assetFilter: (fileName) =>
+          fileName.includes('sqlite3-') &&
+          !fileName.includes('sqlite3-opfs-async-proxy'),
+        expectedCount: 2,
+      }),
+      // Would you believe that there's no other way to do this?
+      {
+        name: 'move-html-files-to-root',
+        generateBundle: {
+          order: 'post',
+          handler(_, bundle) {
+            for (const chunk of Object.values(bundle)) {
+              if (!chunk.fileName.endsWith('.html')) {
+                continue;
+              }
+              chunk.fileName = path.basename(chunk.fileName);
+            }
+          },
+        },
+      },
       // Open the extension in the browser when watching
-      isWatching && extensionDev({ extensionPath: buildDir }),
+      isWatching && extensionDev({ extensionPath: outDir }),
     ],
   };
 });

packages/kernel-browser-runtime/vite.config.ts

@@ -1,7 +1,7 @@
 // eslint-disable-next-line spaced-comment
 /// <reference types="vitest" />
 
-import { jsTrustedPrelude } from '@ocap/vite-plugins';
+import { deduplicateAssets, jsTrustedPrelude } from '@ocap/vite-plugins';
 import type { PreludeRecord } from '@ocap/vite-plugins';
 import path from 'path';
 import { defineConfig } from 'vite';
@@ -62,12 +62,11 @@ export default defineConfig(({ mode }) => {
           vat: path.resolve(sourceDir, 'vat', 'iframe.html'),
         },
         output: {
-          format: 'esm',
           // Basically, create directories for each entry point and put all related
           // files in them.
           entryFileNames: (chunkInfo) => {
-            // This property isn't really documented, but it appears to be equivalent
-            // to the keys of `rollupOptions.input`.
+            // This property isn't really documented, but it appears to be the absolute path
+            // to the entry point.
             if (!chunkInfo.facadeModuleId) {
               return '[name].js';
             }
@@ -85,7 +84,6 @@ export default defineConfig(({ mode }) => {
           },
           chunkFileNames: '[name].js',
           assetFileNames: '[name].[ext]',
-          preserveModulesRoot: sourceDir,
         },
       },
       ...(isDev
@@ -110,28 +108,13 @@ export default defineConfig(({ mode }) => {
       // (It's probably related to the the file being conditionally imported in multiple places.)
       // To avoid bloating the bundle, we delete the duplicate files. Thankfully, these files are
       // extraneous because we don't hit their code paths in practice. (If we did, things would
-      // blow up spectacularly.)
-      {
-        name: 'deduplicate-sqlite-wasm',
-        enforce: 'post',
-        generateBundle(_, bundle) {
-          const extraneousAssets = Object.values(bundle).filter(
-            (assetOrChunk) =>
-              assetOrChunk.fileName.startsWith('sqlite3-') &&
-              !assetOrChunk.fileName.includes('sqlite3-opfs-async-proxy'),
-          );
-
-          if (extraneousAssets.length !== 2) {
-            throw new Error(
-              `Expected 2 extraneous sqlite3.wasm assets, got ${extraneousAssets.length}: ${extraneousAssets.map((asset) => asset.fileName).join(', ')}`,
-            );
-          }
-
-          for (const asset of extraneousAssets) {
-            delete bundle[asset.fileName];
-          }
-        },
-      },
+      // blow up.)
+      deduplicateAssets({
+        assetFilter: (fileName) =>
+          fileName.startsWith('sqlite3-') &&
+          !fileName.includes('sqlite3-opfs-async-proxy'),
+        expectedCount: 2,
+      }),
     ],
   };
 });

packages/vite-plugins/src/deduplicate-assets.ts

@@ -0,0 +1,40 @@
+import type { Plugin as VitePlugin } from 'vite';
+
+type Options = {
+  assetFilter: (fileName: string) => boolean;
+  expectedCount: number;
+};
+
+/**
+ * Vite plugin that deletes extraneous assets from the bundle.
+ *
+ * @param options - Options for the plugin
+ * @param options.assetFilter - A function that filters the assets to be deleted
+ * @param options.expectedCount - The expected number of assets to be deleted
+ * @throws If the number of extraneous assets is not equal to the expected count.
+ * @returns The Vite plugin.
+ */
+export function deduplicateAssets({
+  assetFilter,
+  expectedCount,
+}: Options): VitePlugin {
+  return {
+    name: 'ocap-kernel:deduplicate-assets',
+    enforce: 'post',
+    generateBundle(_, bundle) {
+      const extraneousAssets = Object.values(bundle).filter((assetOrChunk) =>
+        assetFilter(assetOrChunk.fileName),
+      );
+
+      if (extraneousAssets.length !== expectedCount) {
+        throw new Error(
+          `Expected ${expectedCount} extraneous assets, got ${extraneousAssets.length}: ${extraneousAssets.map((asset) => asset.fileName).join(', ')}`,
+        );
+      }
+
+      for (const asset of extraneousAssets) {
+        delete bundle[asset.fileName];
+      }
+    },
+  };
+}

packages/vite-plugins/src/extension-dev.ts

@@ -23,7 +23,7 @@ export function extensionDev({
   };
 
   return {
-    name: 'vite:extension-dev',
+    name: 'ocap-kernel:extension-dev',
 
     // This is called when the server starts
     async configureServer(server) {

packages/vite-plugins/src/html-trusted-prelude.ts

@@ -1,10 +1,10 @@
 import { load as loadHtml } from 'cheerio';
-import path from 'path';
 import { format as prettierFormat } from 'prettier';
 import type { Plugin as VitePlugin } from 'vite';
 
 /**
  * Vite plugin to insert the endoify script before the first script in the head element.
+ * Assumes that `endoify.js` is located in the root of the web app.
  *
  * @throws If the HTML document already references the endoify script or lacks the expected
  * structure.
@@ -13,7 +13,7 @@ import type { Plugin as VitePlugin } from 'vite';
 export function htmlTrustedPrelude(): VitePlugin {
   return {
     name: 'ocap-kernel:html-trusted-prelude',
-    async transformIndexHtml(htmlString, ctx): Promise<string> {
+    async transformIndexHtml(htmlString): Promise<string> {
       const htmlDoc = loadHtml(htmlString);
 
       if (htmlDoc('script[src="endoify.ts"]').length > 0) {
@@ -34,19 +34,7 @@ export function htmlTrustedPrelude(): VitePlugin {
         );
       }
 
-      // Calculate relative path to endoify.js based on HTML file location
-      const htmlFilePath = ctx.filename;
-      const htmlDirPath = path.dirname(htmlFilePath);
-      const rootDir = path.resolve(process.cwd(), 'src');
-      const relativePathToRoot = path.relative(htmlDirPath, rootDir);
-
-      // Create the relative path to endoify.js
-      const endoifyPath = path
-        .join(relativePathToRoot, 'endoify.js')
-        .split(path.sep)
-        .join('/');
-
-      const endoifyElement = `<script src="${endoifyPath}" type="module"></script>`;
+      const endoifyElement = `<script src="/endoify.js" type="module"></script>`;
       htmlDoc(endoifyElement).insertBefore('head:first script:first');
 
       return await prettierFormat(htmlDoc.html(), {

packages/vite-plugins/src/index.test.ts

@@ -5,6 +5,7 @@ import * as indexModule from './index.ts';
 describe('index', () => {
   it('has the expected exports', () => {
     expect(Object.keys(indexModule).sort()).toStrictEqual([
+      'deduplicateAssets',
       'extensionDev',
       'htmlTrustedPrelude',
       'jsTrustedPrelude',