Add Autocomplete and Syntax Highlighting

.gitignore

@@ -117,6 +117,10 @@ test.py
 .yarn/*
 .pnp.cjs
 .pnp.loader.mjs
+
+# Autocomplete generated files
+src/conductor/plugins/autocomplete/builtins/*.json
+
 package-lock.json
 
 .DS_Store

eslint.config.mjs

@@ -10,7 +10,7 @@ export default defineConfig([
   tseslint.configs.recommended,
   eslintConfigPrettierFlat,
   {
-    files: ["src/**/*.{ts,tsx}"],
+    files: ["src/**/*.{ts,tsx,mts}"],
     languageOptions: {
       parser: tseslint.parser,
       parserOptions: {

package.json

@@ -7,16 +7,16 @@
   "scripts": {
     "regen": "tsx src/generate.ts && prettier --write src/ast-types.ts",
     "compile-grammar": "nearleyc src/parser/python.ne | sed -f src/parser/python-grammar.ts.sed |  prettier --stdin-filepath src/parser/python-grammar.ts > src/parser/python-grammar.ts && eslint --fix src/parser/python-grammar.ts",
-    "build": "tsx scripts/build.ts",
-    "dev": "tsx scripts/build.ts --watch",
+    "build": "./scripts/autocomplete.sh && tsx scripts/build.ts",
+    "dev": "./scripts/autocomplete.sh && tsx scripts/build.ts --watch",
     "jsdoc": "./scripts/jsdoc.sh prepare",
     "jsdoc:run": "./scripts/jsdoc.sh run",
     "jsdoc:clean": "./scripts/jsdoc.sh clean",
     "test": "jest",
     "test-coverage": "jest --coverage",
     "lint": "eslint --concurrency=auto src",
-    "format": "prettier --write \"**/*.{ts,tsx,json,js,mjs}\"",
-    "format:ci": "prettier --list-different \"**/*.{ts,tsx,json,js,mjs}\"",
+    "format": "prettier --write \"**/*.{ts,tsx,json,js,mjs,mts}\"",
+    "format:ci": "prettier --list-different \"**/*.{ts,tsx,json,js,mjs,mts}\"",
     "wasm": "tsx src/engines/wasm/runFile.ts"
   },
   "keywords": [
@@ -32,13 +32,15 @@
   "packageManager": "yarn@4.13.0+sha512.5c20ba010c99815433e5c8453112165e673f1c7948d8d2b267f4b5e52097538658388ebc9f9580656d9b75c5cc996f990f611f99304a2197d4c56d21eea370e7",
   "devDependencies": {
     "@inquirer/prompts": "^8.3.2",
+    "@lezer/python": "^1.1.18",
     "@rollup/plugin-commonjs": "^28.0.3",
     "@rollup/plugin-json": "^6.1.0",
     "@rollup/plugin-node-resolve": "^16.0.1",
     "@rollup/plugin-replace": "^6.0.3",
     "@rollup/plugin-terser": "^1.0.0",
     "@rollup/plugin-typescript": "^12.1.2",
     "@rollup/plugin-wasm": "^6.2.2",
+    "@sourceacademy/autocomplete": "github:source-academy/autocomplete#0.0.1",
     "@types/fast-levenshtein": "^0.0.4",
     "@types/jest": "^29.5.14",
     "@types/moo": "^0.5.10",

scripts/autocomplete.sh

@@ -0,0 +1,21 @@
+#! /usr/bin/env bash
+
+# Exit immediately if a command exits with a non-zero status.
+set -e
+
+JSDOC="$(yarn bin jsdoc)"
+LIB="docs/lib"
+CONF="docs/jsdoc/conf.json"
+
+# Create the builtins directory if it doesn't exist
+mkdir -p "src/conductor/plugins/autocomplete/builtins"
+
+# Process every JavaScript file in the docs/lib directory with JSDoc, 
+# outputting the AST as JSON to the src/conductor/plugins/autocomplete/builtins directory.
+for file in "$LIB"/*.js; do
+  echo "Processing $file..."
+  "$JSDOC" -X -c "$CONF" "$file" > "src/conductor/plugins/autocomplete/builtins/$(basename "$file" .js).json" &
+done
+wait
+
+yarn tsx src/generate-autocomplete.mts

src/conductor/PyCseEvaluator.ts

@@ -19,6 +19,7 @@ import pairmutator from "../stdlib/pairmutator";
 import parser from "../stdlib/parser";
 import stream from "../stdlib/stream";
 import { Group } from "../stdlib/utils";
+import AutoCompletePlugin from "./plugins/autocomplete";
 
 function once<T>(fn: () => Promise<T>): () => Promise<T> {
   let promise: Promise<T> | undefined;
@@ -35,11 +36,16 @@ abstract class PyCseEvaluatorBase extends BasicEvaluator {
   private readonly groups: Group[];
   private readonly ensurePreludesLoaded: () => Promise<void>;
 
-  protected constructor(conductor: IRunnerPlugin, variant: number, groups: Group[]) {
+  protected constructor(
+    conductor: IRunnerPlugin,
+    variant: number,
+    groups: Group[],
+    evaluatorName: string,
+  ) {
     super(conductor);
     this.variant = variant;
     this.groups = groups;
-
+    conductor.registerPlugin(AutoCompletePlugin, variant, evaluatorName);
     for (const group of this.groups) {
       for (const [name, value] of group.builtins) {
         this.context.nativeStorage.builtins.set(name, value);
@@ -75,7 +81,6 @@ abstract class PyCseEvaluatorBase extends BasicEvaluator {
       };
 
       await this.ensurePreludesLoaded();
-
       const script = chunk + "\n";
       const ast = parse(script);
       const errors = analyze(
@@ -112,24 +117,29 @@ abstract class PyCseEvaluatorBase extends BasicEvaluator {
 
 export class PyCseEvaluator1 extends PyCseEvaluatorBase {
   constructor(conductor: IRunnerPlugin) {
-    super(conductor, 1, [misc, math]);
+    super(conductor, 1, [misc, math], "PyCseEvaluator1");
   }
 }
 
 export class PyCseEvaluator2 extends PyCseEvaluatorBase {
   constructor(conductor: IRunnerPlugin) {
-    super(conductor, 2, [misc, math, linkedList]);
+    super(conductor, 2, [misc, math, linkedList], "PyCseEvaluator2");
   }
 }
 
 export class PyCseEvaluator3 extends PyCseEvaluatorBase {
   constructor(conductor: IRunnerPlugin) {
-    super(conductor, 3, [misc, math, linkedList, list, pairmutator, stream]);
+    super(conductor, 3, [misc, math, linkedList, list, pairmutator, stream], "PyCseEvaluator3");
   }
 }
 
 export class PyCseEvaluator4 extends PyCseEvaluatorBase {
   constructor(conductor: IRunnerPlugin) {
-    super(conductor, 4, [misc, math, linkedList, list, pairmutator, stream, parser]);
+    super(
+      conductor,
+      4,
+      [misc, math, linkedList, list, pairmutator, stream, parser],
+      "PyCseEvaluator4",
+    );
   }
 }

src/conductor/plugins/autocomplete/highlight-rules.ts

@@ -0,0 +1,227 @@
+/**
+ * Adapted from https://github.com/ajaxorg/ace/blob/master/src/mode/python_highlight_rules.js
+ *
+ * Copyright (c) 2010, Ajax.org B.V.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *    * Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *    * Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *    * Neither the name of Ajax.org B.V. nor the
+ *      names of its contributors may be used to endorse or promote products
+ *      derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+import { AceRules } from "@sourceacademy/autocomplete";
+import math from "../../../stdlib/math";
+import misc from "../../../stdlib/misc";
+import { getIllegalKeywords, getKeywords } from "./keywords";
+
+export default (variant: number) => {
+  const keywords = getKeywords(variant).join("|");
+  const illegalKeywords = getIllegalKeywords(variant).join("|");
+  const stdlibBuiltins = new Map([...math.builtins, ...misc.builtins]);
+  const builtinConstants = [...stdlibBuiltins.keys()]
+    .filter(x => stdlibBuiltins.get(x)?.type !== "builtin")
+    .join("|");
+
+  let builtinFunctions = [...stdlibBuiltins.keys()]
+    .filter(x => stdlibBuiltins.get(x)?.type === "builtin")
+    .join("|");
+  if (variant >= 3) {
+    builtinFunctions += "|range";
+  }
+
+  //var futureReserved = "";
+  const keywordMapper = {
+    map: {
+      "support.function": builtinFunctions,
+      "constant.language": builtinConstants,
+      keyword: keywords,
+      "invalid.illegal": illegalKeywords,
+    },
+    defaultToken: "identifier",
+  };
+
+  const decimalInteger = "(?:(?:[1-9]\\d*)|(?:0))";
+  const octInteger = "(?:0[oO]?[0-7]+)";
+  const hexInteger = "(?:0[xX][\\dA-Fa-f]+)";
+  const binInteger = "(?:0[bB][01]+)";
+  const integer =
+    "(?:" + decimalInteger + "|" + octInteger + "|" + hexInteger + "|" + binInteger + ")";
+
+  const exponent = "(?:[eE][+-]?\\d+)";
+  const fraction = "(?:\\.\\d+)";
+  const intPart = "(?:\\d+)";
+  const pointFloat = "(?:(?:" + intPart + "?" + fraction + ")|(?:" + intPart + "\\.))";
+  const exponentFloat = "(?:(?:" + pointFloat + "|" + intPart + ")" + exponent + ")";
+  const floatNumber = "(?:" + exponentFloat + "|" + pointFloat + ")";
+
+  const stringEscape =
+    "\\\\(x[0-9A-Fa-f]{2}|[0-7]{3}|[\\\\abfnrtv'\"]|U[0-9A-Fa-f]{8}|u[0-9A-Fa-f]{4})";
+  const rules: AceRules = {
+    start: [
+      {
+        token: "comment",
+        regex: "#.*$",
+      },
+      {
+        token: "string", // multi line """ string start
+        regex: '"{3}',
+        next: "qqstring3",
+      },
+      {
+        token: "string", // " string
+        regex: '"(?=.)',
+        next: "qqstring",
+      },
+      {
+        token: "string", // multi line ''' string start
+        regex: "'{3}",
+        next: "qstring3",
+      },
+      {
+        token: "string", // ' string
+        regex: "'(?=.)",
+        next: "qstring",
+      },
+      {
+        token: "keyword.operator",
+        regex: "\\+|\\-|\\*|\\*\\*|\\/|\\/\\/|%|@|<<|>>|&|\\||\\^|~|<|>|<=|=>|==|!=|<>|=",
+      },
+      {
+        token: "punctuation",
+        regex: ",|:|;|\\->|\\+=|\\-=|\\*=|\\/=|\\/\\/=|%=|@=|&=|\\|=|^=|>>=|<<=|\\*\\*=",
+      },
+      {
+        token: "paren.lparen",
+        regex: "[\\[\\(\\{]",
+      },
+      {
+        token: "paren.rparen",
+        regex: "[\\]\\)\\}]",
+      },
+      {
+        token: ["keyword", "text", "entity.name.function"],
+        regex: "(def)(\\s+)([\\u00BF-\\u1FFF\\u2C00-\\uD7FF\\w]+)",
+      },
+      {
+        token: "text",
+        regex: "\\s+",
+      },
+      {
+        include: "constants",
+      },
+    ],
+    qqstring3: [
+      {
+        token: "constant.language.escape",
+        regex: stringEscape,
+      },
+      {
+        token: "string", // multi line """ string end
+        regex: '"{3}',
+        next: "start",
+      },
+      {
+        defaultToken: "string",
+      },
+    ],
+    qstring3: [
+      {
+        token: "constant.language.escape",
+        regex: stringEscape,
+      },
+      {
+        token: "string", // multi line ''' string end
+        regex: "'{3}",
+        next: "start",
+      },
+      {
+        defaultToken: "string",
+      },
+    ],
+    qqstring: [
+      {
+        token: "constant.language.escape",
+        regex: stringEscape,
+      },
+      {
+        token: "string",
+        regex: "\\\\$",
+        next: "qqstring",
+      },
+      {
+        token: "string",
+        regex: '"|$',
+        next: "start",
+      },
+      {
+        defaultToken: "string",
+      },
+    ],
+    qstring: [
+      {
+        token: "constant.language.escape",
+        regex: stringEscape,
+      },
+      {
+        token: "string",
+        regex: "\\\\$",
+        next: "qstring",
+      },
+      {
+        token: "string",
+        regex: "'|$",
+        next: "start",
+      },
+      {
+        defaultToken: "string",
+      },
+    ],
+    constants: [
+      {
+        token: "constant.numeric", // imaginary
+        regex: "(?:" + floatNumber + "|\\d+)[jJ]\\b",
+      },
+      {
+        token: "constant.numeric", // float
+        regex: floatNumber,
+      },
+      {
+        token: "constant.numeric", // long integer
+        regex: integer + "[lL]\\b",
+      },
+      {
+        token: "constant.numeric", // integer
+        regex: integer + "\\b",
+      },
+      {
+        token: ["punctuation", "function.support"], // method
+        regex: "(\\.)([a-zA-Z_]+)\\b",
+      },
+      {
+        token: keywordMapper,
+        regex: "[a-zA-Z_$][a-zA-Z0-9_$]*\\b",
+      },
+    ],
+  };
+
+  return rules;
+};