feat: add scoped runtime (#82)

* add scoped runtime

* add tree-sitter wasm

Author: lihebi

ui/config-overrides.js

@@ -15,5 +15,9 @@ module.exports = function override(config, env) {
   };
   // by default load all the languages
   config.plugins.push(new MonacoWebpackPlugin());
+  config.resolve.fallback = {
+    fs: false,
+    path: false,
+  };
   return config;
 };

ui/package.json

@@ -41,6 +41,7 @@
     "socket.io-client": "^4.5.3",
     "stompjs": "^2.3.3",
     "typescript": "^4.4.2",
+    "web-tree-sitter": "^0.20.7",
     "web-vitals": "^2.1.0",
     "xterm": "^5.0.0",
     "xterm-addon-fit": "^0.6.0",
@@ -88,6 +89,8 @@
     "babel-plugin-named-exports-order": "^0.0.2",
     "prop-types": "^15.8.1",
     "react-app-rewired": "^2.2.1",
+    "tree-sitter-javascript": "^0.19.0",
+    "tree-sitter-python": "^0.20.1",
     "webpack": "^5.74.0"
   }
 }

ui/public/tree-sitter-javascript.wasm

@@ -51,6 +51,7 @@ import { useApolloClient } from "@apollo/client";
 import { CanvasContextMenu } from "./CanvasContextMenu";
 import ToolBox, { ToolTypes } from "./Toolbox";
 import styles from "./canvas.style.js";
+import { analyzeCode } from "../lib/parser";
 
 const nanoid = customAlphabet(nolookalikes, 10);
 
@@ -194,7 +195,6 @@ const ScopeNode = memo<Props>(({ data, id, isConnectable }) => {
 function ResultBlock({ pod, id, showOutput = true }) {
   const store = useContext(RepoContext);
   if (!store) throw new Error("Missing BearContext.Provider in the tree");
-  const wsRun = useStore(store, (state) => state.wsRun);
   return (
     <Box>
       {pod.result && (
@@ -277,6 +277,9 @@ const CodeNode = memo<Props>(({ data, id, isConnectable }) => {
   if (!store) throw new Error("Missing BearContext.Provider in the tree");
   // const pod = useStore(store, (state) => state.pods[id]);
   const wsRun = useStore(store, (state) => state.wsRun);
+  const clearResults = useStore(store, (s) => s.clearResults);
+  const setSymbolTable = useStore(store, (s) => s.setSymbolTable);
+  const setPodVisibility = useStore(store, (s) => s.setPodVisibility);
   const ref = useRef(null);
   const [target, setTarget] = React.useState<any>(null);
   const [frame] = React.useState({
@@ -332,7 +335,18 @@ const CodeNode = memo<Props>(({ data, id, isConnectable }) => {
         deleteNodeById(id);
         break;
       case ToolTypes.play:
-        wsRun(data.id);
+        {
+          // analyze code and set symbol table
+          // TODO maybe put this logic elsewhere?
+          let { ispublic, names } = analyzeCode(pod.content);
+          setPodVisibility(id, ispublic);
+          console.log("names", names);
+          if (names) {
+            setSymbolTable(data.id, names);
+          }
+          clearResults(data.id);
+          wsRun(data.id);
+        }
         break;
       case ToolTypes.layout:
         setLayout(layout === "bottom" ? "right" : "bottom");
@@ -357,11 +371,16 @@ const CodeNode = memo<Props>(({ data, id, isConnectable }) => {
       <Box
         sx={{
           border: "solid 1px #d6dee6",
+          borderWidth: pod.ispublic ? "4px" : "1px",
           borderRadius: "4px",
           width: "100%",
           height: "100%",
           backgroundColor: "rgb(244, 246, 248)",
-          borderColor: isEditorBlur ? "#d6dee6" : "#3182ce",
+          borderColor: pod.ispublic
+            ? "green"
+            : isEditorBlur
+            ? "#d6dee6"
+            : "#3182ce",
         }}
         ref={ref}
       >

ui/public/tree-sitter-python.wasm

@@ -0,0 +1,91 @@
+import Parser from "web-tree-sitter";
+
+let parser: Parser | null = null;
+Parser.init({
+  locateFile(scriptName: string, scriptDirectory: string) {
+    return scriptName;
+  },
+}).then(async () => {
+  /* the library is ready */
+  console.log("tree-sitter is ready");
+  parser = new Parser();
+  const Python = await Parser.Language.load("/tree-sitter-python.wasm");
+  parser.setLanguage(Python);
+});
+
+/**
+ * Return a list of names defined in this code.
+ * @param code the code
+ * @returns a list of names defined in this code.
+ */
+export function analyzeCode(code) {
+  let names: string[] = [];
+  let ispublic = false;
+  if (code.trim().startsWith("@export")) {
+    ispublic = true;
+    code = code.slice(7).trim();
+  }
+  if (!parser) {
+    throw Error("warning: parser not ready");
+  }
+  let tree = parser.parse(code);
+  tree.rootNode.children.forEach((node) => {
+    if (node.type === "function_definition") {
+      let name = node.firstNamedChild!.text;
+      names.push(name);
+    }
+  });
+  return { ispublic, names };
+}
+
+/**
+ * 1. parse the code, get: (defs, refs) to functions & variables
+ * 2. consult symbol table to resolve them
+ * 3. if all resolved, rewrite the code; otherwise, return null.
+ * @param code
+ * @param symbolTable
+ * @returns
+ */
+export function rewriteCode(code, symbolTable) {
+  console.log("--- rewriteCode with symbol table", symbolTable);
+  if (!parser) {
+    throw Error("warning: parser not ready");
+  }
+  let ispublic = false;
+  if (code.trim().startsWith("@export")) {
+    ispublic = true;
+    code = code.slice(7).trim();
+  }
+  let tree = parser.parse(code);
+  // iterate through all first-level children.
+  let defrefs: Parser.SyntaxNode[] = [];
+  // get all the references to variables and functions (currently just functions)
+  let all_query = parser
+    .getLanguage()
+    .query(
+      "[" +
+        "(function_definition (identifier) @name)" +
+        "(call (identifier) @name)" +
+        "(module (expression_statement (identifier) @id))" +
+        "]"
+    );
+  all_query.matches(tree.rootNode).forEach((match) => {
+    let node = match.captures[0].node;
+    defrefs.push(node);
+  });
+  // replace with symbol table
+  let newcode = "";
+  let index = 0;
+  defrefs.forEach((node) => {
+    newcode += code.slice(index, node.startIndex);
+    if (node.text in symbolTable) {
+      newcode += symbolTable[node.text];
+    } else {
+      console.log("warning: cannot resolve", node.text);
+      newcode += node.text;
+    }
+    index = node.endIndex;
+  });
+  newcode += code.slice(index);
+  return { ispublic, newcode };
+}

ui/public/tree-sitter.js

@@ -4,6 +4,7 @@ import { createStore, StateCreator, StoreApi } from "zustand";
 
 // FIXME cyclic import
 import { RepoSlice } from "./store";
+import { rewriteCode } from "./parser";
 
 function getChildExports({ id, pods }) {
   // get all the exports and reexports. The return would be:
@@ -288,6 +289,58 @@ function getExports(content) {
   return { exports, reexports, content };
 }
 
+function doRun({ id, socket, set, get }) {
+  // 1. rewrite the code
+  let pods = get().pods;
+  let pod = pods[id];
+  let code = pod.content;
+  // get symbol tables
+  //
+  // TODO currently, I'm using only the symbol table from the current pod. I'll
+  // need to get the symbol table from all the children as well.
+
+  // I should get symbol table of:
+  // - all sibling nodes
+  console.log("rewriting code ..");
+  // console.log("my id", id, pod.symbolTable);
+  // console.log("=== children", pods[pod.parent].children);
+  // FIXME what if there are conflicts?
+  let allSymbolTables = pods[pod.parent].children.map(({ id, type }) => {
+    // FIXME make this consistent, CODE, POD, DECK, SCOPE; use enums
+    if (pods[id].type === "CODE") {
+      return pods[id].symbolTable || {};
+    } else {
+      let tables = pods[id].children
+        .filter(({ id }) => pods[id].ispublic)
+        .map(({ id }) => pods[id].symbolTable || {});
+      return Object.assign({}, ...tables);
+    }
+  });
+  // console.log("=== allSymbolTables", allSymbolTables);
+  let combinedSymbolTable = Object.assign({}, ...allSymbolTables);
+  // console.log("=== combinedSymbolTable", combinedSymbolTable);
+  let { ispublic, newcode } = rewriteCode(code, combinedSymbolTable);
+  get().setPodVisibility(id, ispublic);
+  console.log("new code:\n", newcode);
+
+  get().setRunning(pod.id);
+  socket.send(
+    JSON.stringify({
+      type: "runCode",
+      payload: {
+        lang: pod.lang,
+        code: newcode,
+        namespace: pod.ns,
+        raw: true,
+        podId: pod.id,
+        sessionId: get().sessionId,
+      },
+    })
+  );
+
+  // 2. send for evaluation
+}
+
 function handleRunTree({ id, socket, set, get }) {
   // get all pods
   function helper(id) {
@@ -540,6 +593,7 @@ export interface RuntimeSlice {
   clearResults: (id) => void;
   clearAllResults: () => void;
   setRunning: (id) => void;
+  setSymbolTable: (id: string, names: string[]) => void;
   addPodExport: (id, exports, reexports) => void;
   clearAllExports: () => void;
   setPodExport: ({ id, exports, reexports }) => void;
@@ -657,7 +711,7 @@ export const createRuntimeSlice: StateCreator<
       });
       return;
     }
-    handleRunTree({
+    doRun({
       id,
       socket: {
         send: (payload) => {
@@ -670,6 +724,7 @@ export const createRuntimeSlice: StateCreator<
     });
   },
   wsPowerRun: ({ id, doEval }) => {
+    throw Error("Depcrecated");
     if (!get().socket) {
       get().addError({
         type: "error",
@@ -721,6 +776,19 @@ export const createRuntimeSlice: StateCreator<
   },
   // ==========
   // exports
+  setSymbolTable: (id: string, names: string[]) => {
+    set(
+      produce((state) => {
+        // a symbol table is foo->foo_<podid>
+        state.pods[id].symbolTable = Object.assign(
+          {},
+          ...names.map((name) => ({
+            [name]: `${name}_${id}`,
+          }))
+        );
+      })
+    );
+  },
   addPodExport: ({ id, name }) => {
     set(
       produce((state) => {

ui/public/tree-sitter.wasm

@@ -17,6 +17,7 @@ import { WebsocketProvider } from "y-websocket";
 import { createRuntimeSlice, RuntimeSlice } from "./runtime";
 import { ApolloClient } from "@apollo/client";
 import { addAwarenessStyle } from "./styles";
+import { analyzeCode } from "./parser";
 
 // Tofix: can't connect to http://codepod.127.0.0.1.sslip.io/socket/, but it works well on webbrowser or curl
 let serverURL;
@@ -111,6 +112,8 @@ export type Pod = {
   fold?: boolean;
   thundar?: boolean;
   utility?: boolean;
+  symbolTable?: { [key: string]: string };
+  ispublic?: boolean;
   exports?: { [key: string]: string[] };
   imports?: {};
   reexports?: {};
@@ -191,6 +194,7 @@ export interface RepoSlice {
   getPod: (string) => Pod;
   getPods: () => Record<string, Pod>;
   getId2children: (string) => string[];
+  setPodVisibility: (id, visible) => void;
 }
 
 type BearState = RepoSlice & RuntimeSlice;
@@ -268,6 +272,8 @@ const createRepoSlice: StateCreator<
       thundar: false,
       utility: false,
       name: "",
+      ispublic: false,
+      symbolTable: {},
       exports: {},
       imports: {},
       reexports: {},
@@ -665,6 +671,13 @@ const createRepoSlice: StateCreator<
         state.pods[id].dirty = true;
       })
     ),
+  setPodVisibility: (id, ispublic) => {
+    set(
+      produce((state) => {
+        state.pods[id].ispublic = ispublic;
+      })
+    );
+  },
   loadRepo: async (client, id) => {
     const { pods, name } = await doRemoteLoadRepo({ id, client });
     set(
@@ -679,6 +692,15 @@ const createRepoSlice: StateCreator<
             state.id2parent[pod.id] = pod.parent.id;
           }
           state.id2children[pod.id] = pod.children.map((child) => child.id);
+          // trigger analyze code for symbol table
+          let { ispublic, names } = analyzeCode(pod.content);
+          pod.ispublic = ispublic;
+          pod.symbolTable = Object.assign(
+            {},
+            ...names.map((name) => ({
+              [name]: `${name}_${id}`,
+            }))
+          );
         }
         state.repoLoaded = true;
       })