hpcc-systems · GordonSmith · Aug 25, 2025 · Aug 7, 2025 · Aug 8, 2025 · Aug 14, 2025
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/comms/package.json b/packages/comms/package.json
@@ -91,7 +91,7 @@
         "d3-time-format": "^2",
         "data-uri-to-buffer": "6.0.2",
         "safe-buffer": "5.2.1",
-        "tmp": "0.2.3",
+        "tmp": "0.2.4",
         "soap": "1.2.0",
         "typescript-formatter": "^7.2.2"
     },

diff --git a/packages/markdown-it-plugins/package.json b/packages/markdown-it-plugins/package.json
@@ -54,7 +54,10 @@
     "dependencies": {
         "@hpcc-js/observablehq-compiler": "^3.3.9",
         "@observablehq/framework": "1.13.3",
-        "@observablehq/runtime": "5.9.9",
+        "@observablehq/parser": "6.1.0",
+        "@observablehq/inspector": "5.0.1",
+        "@observablehq/stdlib": "5.8.8",
+        "@observablehq/runtime": "6.0.0",
         "@types/markdown-it": "14.1.2"
     },
     "peerDependencies": {

diff --git a/packages/observablehq-compiler/.vscode/launch.json b/packages/observablehq-compiler/.vscode/launch.json
@@ -43,6 +43,20 @@
                 "!**/node_modules/**"
             ]
         },
+        {
+            "name": "index-notebook-kit.html",
+            "request": "launch",
+            "type": "msedge",
+            "url": "http://localhost:5514/index-notebook-kit.html",
+            "runtimeArgs": [
+                "--disable-web-security"
+            ],
+            "webRoot": "${workspaceFolder}",
+            "outFiles": [
+                "${workspaceFolder}/**/*.js",
+                "!**/node_modules/**"
+            ]
+        },
         {
             "name": "index-preview.html",
             "request": "launch",

diff --git a/packages/observablehq-compiler/README.md b/packages/observablehq-compiler/README.md
@@ -80,7 +80,9 @@ The interpreter is 100% compatible with:
 It also supports fetching notebooks directly from the ObservableHQ repository.
 
 ```js
-import { Library, Runtime, Inspector } from "@observablehq/runtime";
+import { Inspector } from "@observablehq/inspector";
+import { Library } from "@observablehq/stdlib";
+import { Runtime } from "@observablehq/runtime";
 import { download, compile } from "@hpcc-js/observablehq-compiler";
 
 const placeholder = document.getElementById("placeholder");

diff --git a/packages/observablehq-compiler/esbuild.js b/packages/observablehq-compiler/esbuild.js
diff --git a/packages/observablehq-compiler/index-notebook-kit.html b/packages/observablehq-compiler/index-notebook-kit.html
@@ -0,0 +1,39 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <title>Home</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            margin: 0;
+            padding: 0;
+            background-color: #f0f0f0;
+        }
+
+        h1 {
+            text-align: center;
+            margin-top: 50px;
+        }
+
+        #placeholder {
+            width: 100%;
+            height: 500px;
+            background-color: #fff;
+            margin-top: 20px;
+        }
+    </style>
+</head>
+
+<body>
+    <h1>ESM Quick Test</h1>
+    <div id="placeholder"></div>
+    <script type="module">
+
+        import { test } from "./tests/index-notebook-kit.ts";
+        test();
+
+    </script>
+</body>
+
+</html>
diff --git a/packages/observablehq-compiler/index-preview.html b/packages/observablehq-compiler/index-preview.html
@@ -26,7 +26,14 @@
     <script type="importmap">
         {
             "imports": {
-                "@observablehq/runtime": "https://cdn.jsdelivr.net/npm/@observablehq/runtime@5.9.9/dist/runtime.js",
+                "@observablehq/stdlib": "https://cdn.jsdelivr.net/npm/@observablehq/stdlib@5.8.8/src/index.js",
+                "@observablehq/inspector": "https://cdn.jsdelivr.net/npm/@observablehq/inspector@5.0.1/src/index.js",
+                "@observablehq/runtime": "https://cdn.jsdelivr.net/npm/@observablehq/runtime@6.0.0/src/index.js",
+                "isoformat": "https://cdn.jsdelivr.net/npm/isoformat@0.2.1/src/index.js",
+                "d3-dsv": "https://cdn.jsdelivr.net/npm/d3-dsv@3.0.1/src/index.js",
+                "d3-require": "https://cdn.jsdelivr.net/npm/d3-require@1.3.0/src/index.mjs",
+                "d3-array": "https://cdn.jsdelivr.net/npm/d3-array@3.2.4/src/index.js",
+                "internmap": "https://cdn.jsdelivr.net/npm/internmap@2.0.3/src/index.js",
 
                 "@hpcc-js/observablehq-compiler": "../observablehq-compiler/dist/index.js"
             }
@@ -38,7 +45,9 @@
     <h1>ESM Quick Test</h1>
     <div id="placeholder"></div>
     <script type="module">
-        import { Library, Runtime, Inspector } from "@observablehq/runtime";
+        import { Inspector } from "@observablehq/inspector";
+        import { Library } from "@observablehq/stdlib";
+        import { Runtime } from "@observablehq/runtime";
         import { omd2notebook, compile } from "@hpcc-js/observablehq-compiler";
 
         const md = `\

diff --git a/packages/observablehq-compiler/index.html b/packages/observablehq-compiler/index.html
@@ -29,7 +29,9 @@
     <h1>ESM Quick Test</h1>
     <div id="placeholder"></div>
     <script type="module">
-        import { Library, Runtime, Inspector } from "@observablehq/runtime";
+        import { Inspector } from "@observablehq/inspector";
+        import { Library } from "@observablehq/stdlib";
+        import { Runtime } from "@observablehq/runtime";
         import { omd2notebook, compile } from "./src/index.ts";
 
         const md = `\

diff --git a/packages/observablehq-compiler/package.json b/packages/observablehq-compiler/package.json
@@ -48,7 +48,10 @@
     "devDependencies": {
         "@hpcc-js/esbuild-plugins": "^1.4.9",
         "@observablehq/parser": "6.1.0",
-        "@observablehq/runtime": "5.9.9"
+        "@observablehq/inspector": "5.0.1",
+        "@observablehq/stdlib": "5.8.8",
+        "@observablehq/notebook-kit": "1.0.1"
+
     },
     "repository": {
         "type": "git",

diff --git a/packages/observablehq-compiler/src/index.ts b/packages/observablehq-compiler/src/index.ts
@@ -1,7 +1,10 @@
-export type { ohq } from "./observable-shim.ts";
+export type { ohq } from "./ojs/observable-shim.ts";
 
-export * from "./compiler.ts";
-export { ojs2notebook, omd2notebook, download } from "./util.ts";
-export * from "./writer.ts";
+export * from "./ojs/compiler.ts";
+export { ojs2notebook, omd2notebook } from "./ojs/util.ts";
+export { download } from "./util/comms.ts";
+export * from "./ojs/writer.ts";
 
 import "../src/index.css";
+
+export * from "./ohqnk/index.ts";
diff --git a/packages/observablehq-compiler/src/ohqnk/compiler.ts b/packages/observablehq-compiler/src/ohqnk/compiler.ts
@@ -0,0 +1,148 @@
+import { Identifier } from "acorn";
+import { FileAttachments } from "@observablehq/stdlib";
+import { type Notebook, type Cell, parseJavaScript, transpile, type TranspiledJavaScript } from "@observablehq/notebook-kit";
+import { type DefineState, type Definition } from "@observablehq/notebook-kit/runtime";
+import type { Inspector } from "@observablehq/inspector";
+import type { Runtime, Module } from "@observablehq/runtime";
+import { define } from "@observablehq/notebook-kit/runtime";
+import { fixRelativeUrl } from "../util/paths.ts";
+import { fetchEx } from "../util/comms.ts";
+import { html2notebook } from "./util.ts";
+
+export type InspectorFactory = (name: string | undefined, id: string | number) => Inspector;
+
+const FunctionConstructors = {
+    regular: Object.getPrototypeOf(function () { }).constructor,
+    async: Object.getPrototypeOf(async function () { }).constructor,
+    generator: Object.getPrototypeOf(function* () { }).constructor,
+    asyncGenerator: Object.getPrototypeOf(async function* () { }).constructor,
+} as const;
+
+type FunctionConstructor =
+    | typeof FunctionConstructors.regular
+    | typeof FunctionConstructors.async
+    | typeof FunctionConstructors.generator
+    | typeof FunctionConstructors.asyncGenerator;
+
+type FunctionLikeExpression = {
+    type: "FunctionExpression" | "ArrowFunctionExpression";
+    body: { type: "BlockStatement" | "Expression"; start: number; end: number };
+    async?: boolean;
+    generator?: boolean;
+    params?: Identifier[];
+};
+
+function constructFunction(input: string) {
+    const { body } = parseJavaScript(input) as { body: FunctionLikeExpression };
+    if (body.type !== "FunctionExpression" && body.type !== "ArrowFunctionExpression") {
+        throw new Error(`Unsupported function type: ${body.type}`);
+    }
+
+    const Constructor = (body.async && body.generator) ?
+        FunctionConstructors.asyncGenerator :
+        body.async ?
+            FunctionConstructors.async :
+            body.generator ?
+                FunctionConstructors.generator :
+                FunctionConstructors.regular;
+
+    const params = body.params?.map(param => input.substring(param.start, param.end)).join(", ") || "";
+    const bodyStr = body.body.type === "BlockStatement" ? input.substring(body.body.start, body.body.end).slice(1, -1).trim() : input.substring(body.body.start, body.body.end);
+    return Constructor(params, bodyStr);
+}
+interface CellEx extends Cell {
+    transpiled: TranspiledJavaScript;
+}
+function transpileCell(cell: Cell): CellEx {
+    const retVal: CellEx = {
+        ...cell,
+        transpiled: transpile(cell.value, cell.mode)
+    };
+    retVal.transpiled.body = constructFunction(retVal.transpiled.body);
+    return retVal;
+}
+export interface CompileOptions {
+    baseUrl?: string;
+    importMode?: "recursive" | "precompiled";
+}
+export function notebook(_cells: CellEx[] = [], { baseUrl = ".", importMode = "precompiled" }: CompileOptions = {}) {
+    const stateById = new Map<number, DefineState>();
+
+    function add(vscodeCellID: number, definition: Definition, inspector: Inspector): void {
+        let state = stateById.get(vscodeCellID);
+        if (state) {
+            state.variables.forEach((v) => v.delete());
+            state.variables = [];
+        } else {
+            state = { root: inspector._node, expanded: [], variables: [] };
+            stateById.set(vscodeCellID, state);
+        }
+        define(state, definition);
+    }
+
+    function remove(vscodeCellID: number): void {
+        const state = stateById.get(vscodeCellID)!;
+        state.root.remove();
+        state.variables.forEach((v) => v.delete());
+        stateById.delete(vscodeCellID);
+    }
+
+    function removeAll(): void {
+        const keys = Array.from(stateById.keys());
+        for (const key of keys) {
+            remove(key);
+        }
+    }
+
+    const files: any[] = [];
+    const fileAttachmentsMap = new Map<string, any>(files);
+    const cells = new Map<number, CellEx>(_cells.map(c => [c.id, c]));
+
+    const retVal = (runtime: Runtime, inspectorFactory: InspectorFactory): Module => {
+        runtime.main.builtin("fetchEx", fetchEx);
+        cells.forEach(cell => {
+            const inspector = inspectorFactory(`cell-${cell.id}`, cell.id);
+            add(cell.id, { id: cell.id, ...cell.transpiled }, inspector);
+        });
+        return runtime.main;
+    };
+    retVal.fileAttachments = fileAttachmentsMap;
+    retVal.cells = cells;
+    // retVal.set = async (n: Cell): Promise<JavaScriptCell> => {
+    //     const cell = await transpileCell(n, { baseUrl, importMode });
+    //     retVal.delete(cell.id);
+    //     cells.set(cell.id, cell);
+    //     return cell;
+    // };
+    // retVal.get = (id: string | number): JavaScriptCell | undefined => {
+    //     return cells.get(id);
+    // };
+    // retVal.delete = (id: string | number): boolean => {
+    //     const cell = cells.get(id);
+    //     if (cell) {
+    //         cell.delete();
+    //         return cells.delete(id);
+    //     }
+    //     return false;
+    // };
+    // retVal.clear = () => {
+    //     cells.forEach(cell => cell.delete());
+    //     cells.clear();
+    // };
+    // retVal.write = (w: Writer) => {
+    //     w.files(_files);
+    //     cells.forEach(cell => cell.write(w));
+    // };
+    // retVal.toString = (w = new Writer()) => {
+    //     retVal.write(w);
+    //     return w.toString().trim();
+    // };
+    return retVal;
+}
+
+export function compile(notebookOrOjs: Notebook | string, { baseUrl = ".", importMode = "precompiled" }: CompileOptions = {}) {
+    const htmlNotebook = typeof notebookOrOjs === "string" ? html2notebook(notebookOrOjs) : notebookOrOjs;
+    const _cells: CellEx[] = htmlNotebook.cells.map(n => transpileCell(n));
+    return notebook(_cells, { baseUrl, importMode });
+}
+export type compileFunc = Awaited<ReturnType<typeof compile>>;
diff --git a/packages/observablehq-compiler/src/ohqnk/index.ts b/packages/observablehq-compiler/src/ohqnk/index.ts
@@ -0,0 +1,14 @@
+import type { Notebook as ohqnkNotebook, Cell as ohqnkCell } from "@observablehq/notebook-kit";
+import { compile as ohqnkCompile } from "./compiler.ts";
+import { html2notebook as ohqnkHtml2notebook } from "./util.ts";
+
+export namespace ohqnk {
+    export interface Notebook extends ohqnkNotebook {
+    }
+
+    export interface Cell extends ohqnkCell {
+    }
+
+    export const compile = ohqnkCompile;
+    export const html2notebook = ohqnkHtml2notebook;
+}
diff --git a/packages/observablehq-compiler/src/ohqnk/util.ts b/packages/observablehq-compiler/src/ohqnk/util.ts
@@ -0,0 +1,5 @@
+import { deserialize, type Notebook } from "@observablehq/notebook-kit";
+
+export function html2notebook(html: string): Notebook {
+    return deserialize(html);
+}
diff --git a/...ges/observablehq-compiler/src/compiler.md → ...observablehq-compiler/src/ojs/compiler.md b/...ges/observablehq-compiler/src/compiler.md → ...observablehq-compiler/src/ojs/compiler.md
@@ -71,7 +71,9 @@ To render it to a web page, simply follow the same steps as if you had downloade
 
 ```js
 
-import { Library, Runtime, Inspector } from "https://cdn.skypack.dev/@observablehq/runtime";
+import { Inspector } from "https://cdn.skypack.dev/@observablehq/inspector";
+import { Library } from "https://cdn.skypack.dev/@observablehq/stdlib";
+import { Runtime } from "https://cdn.skypack.dev/@observablehq/runtime";
 
 const placeholder = document.getElementById("placeholder");
 
@@ -93,7 +95,9 @@ Putting it all together:
     <div id="placeholder" style="height:100%;overflow-y:scroll">
     </div>
     <script type="module">
-      import { Library, Runtime, Inspector } from "https://cdn.skypack.dev/@observablehq/runtime";
+      import { Inspector } from "https://cdn.skypack.dev/@observablehq/inspector";
+      import { Library } from "https://cdn.skypack.dev/@observablehq/stdlib";
+      import { Runtime } from "https://cdn.skypack.dev/@observablehq/runtime";
       import { compile } from "@hpcc-js/observablehq-compiler";
 
       const placeholder = document.getElementById("placeholder");
@@ -183,7 +187,9 @@ To output the generated code simply call `toString` on the compiled function:
     <div id="placeholder" style="height:400px;overflow-y:scroll">
     </div>
     <script type="module">
-      import { Library, Runtime, Inspector } from "https://cdn.skypack.dev/@observablehq/runtime";
+      import { Inspector } from "https://cdn.skypack.dev/@observablehq/inspector";
+      import { Library } from "https://cdn.skypack.dev/@observablehq/stdlib";
+      import { Runtime } from "https://cdn.skypack.dev/@observablehq/runtime";
       import { download, compile } from "@hpcc-js/observablehq-compiler";
 
       const notebookUrl = "https://observablehq.com/@observablehq/plot";
@@ -212,7 +218,10 @@ To output the generated code simply call `toString` on the compiled function:
     <div id="placeholder" style="height:400px;overflow-y:scroll">
     </div>
     <script type="module">
-      import { Library, Runtime, Inspector } from "https://cdn.skypack.dev/@observablehq/runtime";
+      import { Inspector } from "https://cdn.skypack.dev/@observablehq/inspector";
+      import { Library } from "https://cdn.skypack.dev/@observablehq/stdlib";
+      import { Runtime } from "https://cdn.skypack.dev/@observablehq/runtime";
+
       import { download, compile } from "@hpcc-js/observablehq-compiler";
 
       const notebookUrl = "https://observablehq.com/@mbostock/fullscreen-canvas";