diff --git a/.gitignore b/.gitignore index 6164b23e..f872429a 100644 --- a/.gitignore +++ b/.gitignore @@ -117,6 +117,10 @@ test.py .yarn/* .pnp.cjs .pnp.loader.mjs + +# Autocomplete generated files +src/conductor/plugins/autocomplete/builtins/*.json + package-lock.json .DS_Store diff --git a/eslint.config.mjs b/eslint.config.mjs index b5ccd40a..31ab45d1 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -10,7 +10,7 @@ export default defineConfig([ tseslint.configs.recommended, eslintConfigPrettierFlat, { - files: ["src/**/*.{ts,tsx}"], + files: ["src/**/*.{ts,tsx,mts}"], languageOptions: { parser: tseslint.parser, parserOptions: { diff --git a/package.json b/package.json index 33c7789f..b833d184 100644 --- a/package.json +++ b/package.json @@ -7,16 +7,16 @@ "scripts": { "regen": "tsx src/generate.ts && prettier --write src/ast-types.ts", "compile-grammar": "nearleyc src/parser/python.ne | sed -f src/parser/python-grammar.ts.sed | prettier --stdin-filepath src/parser/python-grammar.ts > src/parser/python-grammar.ts && eslint --fix src/parser/python-grammar.ts", - "build": "tsx scripts/build.ts", - "dev": "tsx scripts/build.ts --watch", + "build": "./scripts/autocomplete.sh && tsx scripts/build.ts", + "dev": "./scripts/autocomplete.sh && tsx scripts/build.ts --watch", "jsdoc": "./scripts/jsdoc.sh prepare", "jsdoc:run": "./scripts/jsdoc.sh run", "jsdoc:clean": "./scripts/jsdoc.sh clean", "test": "jest", "test-coverage": "jest --coverage", "lint": "eslint --concurrency=auto src", - "format": "prettier --write \"**/*.{ts,tsx,json,js,mjs}\"", - "format:ci": "prettier --list-different \"**/*.{ts,tsx,json,js,mjs}\"", + "format": "prettier --write \"**/*.{ts,tsx,json,js,mjs,mts}\"", + "format:ci": "prettier --list-different \"**/*.{ts,tsx,json,js,mjs,mts}\"", "wasm": "tsx src/engines/wasm/runFile.ts" }, "keywords": [ @@ -32,6 +32,7 @@ "packageManager": "yarn@4.13.0+sha512.5c20ba010c99815433e5c8453112165e673f1c7948d8d2b267f4b5e52097538658388ebc9f9580656d9b75c5cc996f990f611f99304a2197d4c56d21eea370e7", "devDependencies": { "@inquirer/prompts": "^8.3.2", + "@lezer/python": "^1.1.18", "@rollup/plugin-commonjs": "^28.0.3", "@rollup/plugin-json": "^6.1.0", "@rollup/plugin-node-resolve": "^16.0.1", @@ -39,6 +40,7 @@ "@rollup/plugin-terser": "^1.0.0", "@rollup/plugin-typescript": "^12.1.2", "@rollup/plugin-wasm": "^6.2.2", + "@sourceacademy/autocomplete": "github:source-academy/autocomplete#0.0.1", "@types/fast-levenshtein": "^0.0.4", "@types/jest": "^29.5.14", "@types/moo": "^0.5.10", diff --git a/scripts/autocomplete.sh b/scripts/autocomplete.sh new file mode 100755 index 00000000..5d93838a --- /dev/null +++ b/scripts/autocomplete.sh @@ -0,0 +1,21 @@ +#! /usr/bin/env bash + +# Exit immediately if a command exits with a non-zero status. +set -e + +JSDOC="$(yarn bin jsdoc)" +LIB="docs/lib" +CONF="docs/jsdoc/conf.json" + +# Create the builtins directory if it doesn't exist +mkdir -p "src/conductor/plugins/autocomplete/builtins" + +# Process every JavaScript file in the docs/lib directory with JSDoc, +# outputting the AST as JSON to the src/conductor/plugins/autocomplete/builtins directory. +for file in "$LIB"/*.js; do + echo "Processing $file..." + "$JSDOC" -X -c "$CONF" "$file" > "src/conductor/plugins/autocomplete/builtins/$(basename "$file" .js).json" & +done +wait + +yarn tsx src/generate-autocomplete.mts diff --git a/src/conductor/PyCseEvaluator.ts b/src/conductor/PyCseEvaluator.ts index 77bfab5d..ce3589fb 100644 --- a/src/conductor/PyCseEvaluator.ts +++ b/src/conductor/PyCseEvaluator.ts @@ -19,6 +19,7 @@ import pairmutator from "../stdlib/pairmutator"; import parser from "../stdlib/parser"; import stream from "../stdlib/stream"; import { Group } from "../stdlib/utils"; +import AutoCompletePlugin from "./plugins/autocomplete"; function once(fn: () => Promise): () => Promise { let promise: Promise | undefined; @@ -35,11 +36,16 @@ abstract class PyCseEvaluatorBase extends BasicEvaluator { private readonly groups: Group[]; private readonly ensurePreludesLoaded: () => Promise; - protected constructor(conductor: IRunnerPlugin, variant: number, groups: Group[]) { + protected constructor( + conductor: IRunnerPlugin, + variant: number, + groups: Group[], + evaluatorName: string, + ) { super(conductor); this.variant = variant; this.groups = groups; - + conductor.registerPlugin(AutoCompletePlugin, variant, evaluatorName); for (const group of this.groups) { for (const [name, value] of group.builtins) { this.context.nativeStorage.builtins.set(name, value); @@ -75,7 +81,6 @@ abstract class PyCseEvaluatorBase extends BasicEvaluator { }; await this.ensurePreludesLoaded(); - const script = chunk + "\n"; const ast = parse(script); const errors = analyze( @@ -112,24 +117,29 @@ abstract class PyCseEvaluatorBase extends BasicEvaluator { export class PyCseEvaluator1 extends PyCseEvaluatorBase { constructor(conductor: IRunnerPlugin) { - super(conductor, 1, [misc, math]); + super(conductor, 1, [misc, math], "PyCseEvaluator1"); } } export class PyCseEvaluator2 extends PyCseEvaluatorBase { constructor(conductor: IRunnerPlugin) { - super(conductor, 2, [misc, math, linkedList]); + super(conductor, 2, [misc, math, linkedList], "PyCseEvaluator2"); } } export class PyCseEvaluator3 extends PyCseEvaluatorBase { constructor(conductor: IRunnerPlugin) { - super(conductor, 3, [misc, math, linkedList, list, pairmutator, stream]); + super(conductor, 3, [misc, math, linkedList, list, pairmutator, stream], "PyCseEvaluator3"); } } export class PyCseEvaluator4 extends PyCseEvaluatorBase { constructor(conductor: IRunnerPlugin) { - super(conductor, 4, [misc, math, linkedList, list, pairmutator, stream, parser]); + super( + conductor, + 4, + [misc, math, linkedList, list, pairmutator, stream, parser], + "PyCseEvaluator4", + ); } } diff --git a/src/conductor/plugins/autocomplete/highlight-rules.ts b/src/conductor/plugins/autocomplete/highlight-rules.ts new file mode 100644 index 00000000..0e6a9a5f --- /dev/null +++ b/src/conductor/plugins/autocomplete/highlight-rules.ts @@ -0,0 +1,227 @@ +/** + * Adapted from https://github.com/ajaxorg/ace/blob/master/src/mode/python_highlight_rules.js + * + * Copyright (c) 2010, Ajax.org B.V. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Ajax.org B.V. nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +import { AceRules } from "@sourceacademy/autocomplete"; +import math from "../../../stdlib/math"; +import misc from "../../../stdlib/misc"; +import { getIllegalKeywords, getKeywords } from "./keywords"; + +export default (variant: number) => { + const keywords = getKeywords(variant).join("|"); + const illegalKeywords = getIllegalKeywords(variant).join("|"); + const stdlibBuiltins = new Map([...math.builtins, ...misc.builtins]); + const builtinConstants = [...stdlibBuiltins.keys()] + .filter(x => stdlibBuiltins.get(x)?.type !== "builtin") + .join("|"); + + let builtinFunctions = [...stdlibBuiltins.keys()] + .filter(x => stdlibBuiltins.get(x)?.type === "builtin") + .join("|"); + if (variant >= 3) { + builtinFunctions += "|range"; + } + + //var futureReserved = ""; + const keywordMapper = { + map: { + "support.function": builtinFunctions, + "constant.language": builtinConstants, + keyword: keywords, + "invalid.illegal": illegalKeywords, + }, + defaultToken: "identifier", + }; + + const decimalInteger = "(?:(?:[1-9]\\d*)|(?:0))"; + const octInteger = "(?:0[oO]?[0-7]+)"; + const hexInteger = "(?:0[xX][\\dA-Fa-f]+)"; + const binInteger = "(?:0[bB][01]+)"; + const integer = + "(?:" + decimalInteger + "|" + octInteger + "|" + hexInteger + "|" + binInteger + ")"; + + const exponent = "(?:[eE][+-]?\\d+)"; + const fraction = "(?:\\.\\d+)"; + const intPart = "(?:\\d+)"; + const pointFloat = "(?:(?:" + intPart + "?" + fraction + ")|(?:" + intPart + "\\.))"; + const exponentFloat = "(?:(?:" + pointFloat + "|" + intPart + ")" + exponent + ")"; + const floatNumber = "(?:" + exponentFloat + "|" + pointFloat + ")"; + + const stringEscape = + "\\\\(x[0-9A-Fa-f]{2}|[0-7]{3}|[\\\\abfnrtv'\"]|U[0-9A-Fa-f]{8}|u[0-9A-Fa-f]{4})"; + const rules: AceRules = { + start: [ + { + token: "comment", + regex: "#.*$", + }, + { + token: "string", // multi line """ string start + regex: '"{3}', + next: "qqstring3", + }, + { + token: "string", // " string + regex: '"(?=.)', + next: "qqstring", + }, + { + token: "string", // multi line ''' string start + regex: "'{3}", + next: "qstring3", + }, + { + token: "string", // ' string + regex: "'(?=.)", + next: "qstring", + }, + { + token: "keyword.operator", + regex: "\\+|\\-|\\*|\\*\\*|\\/|\\/\\/|%|@|<<|>>|&|\\||\\^|~|<|>|<=|=>|==|!=|<>|=", + }, + { + token: "punctuation", + regex: ",|:|;|\\->|\\+=|\\-=|\\*=|\\/=|\\/\\/=|%=|@=|&=|\\|=|^=|>>=|<<=|\\*\\*=", + }, + { + token: "paren.lparen", + regex: "[\\[\\(\\{]", + }, + { + token: "paren.rparen", + regex: "[\\]\\)\\}]", + }, + { + token: ["keyword", "text", "entity.name.function"], + regex: "(def)(\\s+)([\\u00BF-\\u1FFF\\u2C00-\\uD7FF\\w]+)", + }, + { + token: "text", + regex: "\\s+", + }, + { + include: "constants", + }, + ], + qqstring3: [ + { + token: "constant.language.escape", + regex: stringEscape, + }, + { + token: "string", // multi line """ string end + regex: '"{3}', + next: "start", + }, + { + defaultToken: "string", + }, + ], + qstring3: [ + { + token: "constant.language.escape", + regex: stringEscape, + }, + { + token: "string", // multi line ''' string end + regex: "'{3}", + next: "start", + }, + { + defaultToken: "string", + }, + ], + qqstring: [ + { + token: "constant.language.escape", + regex: stringEscape, + }, + { + token: "string", + regex: "\\\\$", + next: "qqstring", + }, + { + token: "string", + regex: '"|$', + next: "start", + }, + { + defaultToken: "string", + }, + ], + qstring: [ + { + token: "constant.language.escape", + regex: stringEscape, + }, + { + token: "string", + regex: "\\\\$", + next: "qstring", + }, + { + token: "string", + regex: "'|$", + next: "start", + }, + { + defaultToken: "string", + }, + ], + constants: [ + { + token: "constant.numeric", // imaginary + regex: "(?:" + floatNumber + "|\\d+)[jJ]\\b", + }, + { + token: "constant.numeric", // float + regex: floatNumber, + }, + { + token: "constant.numeric", // long integer + regex: integer + "[lL]\\b", + }, + { + token: "constant.numeric", // integer + regex: integer + "\\b", + }, + { + token: ["punctuation", "function.support"], // method + regex: "(\\.)([a-zA-Z_]+)\\b", + }, + { + token: keywordMapper, + regex: "[a-zA-Z_$][a-zA-Z0-9_$]*\\b", + }, + ], + }; + + return rules; +}; diff --git a/src/conductor/plugins/autocomplete/index.ts b/src/conductor/plugins/autocomplete/index.ts new file mode 100644 index 00000000..a48c35ab --- /dev/null +++ b/src/conductor/plugins/autocomplete/index.ts @@ -0,0 +1,37 @@ +import { parser } from "@lezer/python"; +import { + AutoCompleteEntry, + BaseAutoCompleteRunnerPlugin, + SyntaxHighlightData, +} from "@sourceacademy/autocomplete"; +import { IChannel, IConduit } from "@sourceacademy/conductor/conduit"; +import pythonMode from "./mode"; +import { getNames } from "./resolver"; + +export default class AutoCompletePlugin extends BaseAutoCompleteRunnerPlugin { + private readonly variant: number; + private readonly evaluatorName: string; + + constructor( + _conduit: IConduit, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + channels: IChannel[], + variant: number, + evaluatorName: string, + ) { + super(_conduit, channels); + this.variant = variant; + this.evaluatorName = evaluatorName; + } + + get mode(): SyntaxHighlightData { + return pythonMode(this.variant, this.evaluatorName); + } + + autocomplete(code: string, row: number, column: number): AutoCompleteEntry[] { + const tree = parser.parse(code); + + const entries = getNames(tree, code, row, column, this.variant); + return entries; + } +} diff --git a/src/conductor/plugins/autocomplete/keywords.ts b/src/conductor/plugins/autocomplete/keywords.ts new file mode 100644 index 00000000..c801d3ec --- /dev/null +++ b/src/conductor/plugins/autocomplete/keywords.ts @@ -0,0 +1,48 @@ +export const getKeywords = (variant: number): string[] => { + let keywords = [ + "and", + "def", + "elif", + "else", + "from", + "global", + "if", + "import", + "lambda", + "not", + "or", + "pass", + "return", + "nonlocal", + ]; + + if (variant >= 3) { + keywords = keywords.concat(["while", "for", "in", "break", "continue"]); + } + return keywords; +}; + +export const getIllegalKeywords = (variant: number): string[] => { + let illegalKeywords = [ + "as", + "assert", + "class", + "del", + "except", + "finally", + "is", + "match", + "case", + "raise", + "try", + "with", + "yield", + "async", + "await", + ]; + + if (variant < 3) { + illegalKeywords = illegalKeywords.concat(["while", "for", "break", "continue", "in"]); + } + return illegalKeywords; +}; diff --git a/src/conductor/plugins/autocomplete/mode.ts b/src/conductor/plugins/autocomplete/mode.ts new file mode 100644 index 00000000..3ff53653 --- /dev/null +++ b/src/conductor/plugins/autocomplete/mode.ts @@ -0,0 +1,25 @@ +import { SyntaxHighlightData } from "@sourceacademy/autocomplete"; +import PythonHighlightRules from "./highlight-rules"; +export default (variant: number, evaluatorName: string): SyntaxHighlightData => ({ + highlightRules: PythonHighlightRules(variant), + foldingRules: { + hookFrom: "ace/mode/folding/pythonic", + args: ["\\:"], + }, + lineCommentStart: "#", + pairQuotesAfter: { + "'": /[ruf]/i, + '"': /[ruf]/i, + }, + indents: { + hookFrom: "ace/mode/python", + }, + outdents: { + hookFrom: "ace/mode/python", + }, + autoOutdent: { + hookFrom: "ace/mode/python", + }, + id: `ace/mode/${evaluatorName}`, + snippetFileId: `ace/snippets/${evaluatorName}`, +}); diff --git a/src/conductor/plugins/autocomplete/resolver.ts b/src/conductor/plugins/autocomplete/resolver.ts new file mode 100644 index 00000000..6a7ea240 --- /dev/null +++ b/src/conductor/plugins/autocomplete/resolver.ts @@ -0,0 +1,240 @@ +import { SyntaxNode, Tree, TreeCursor } from "@lezer/common"; +import { AutoCompleteEntry, CompletionItemKind } from "@sourceacademy/autocomplete"; + +import linkedListJSON from "./builtins/linked_list.json"; +import listJSON from "./builtins/list.json"; +import mathJSON from "./builtins/math.json"; +import miscJSON from "./builtins/misc.json"; +import pairmutatorJSON from "./builtins/pairmutator.json"; +import streamJSON from "./builtins/stream.json"; +import { getKeywords } from "./keywords"; + +type Environment = { + variables: string[]; + functions: string[]; + child: Environment | null; +}; + +const getNodeText = (node: SyntaxNode, doc: string): string => { + return doc.slice(node.from, node.to); +}; + +function isCompletionItemKind(value: string): value is CompletionItemKind { + return Object.values(CompletionItemKind).includes(value as CompletionItemKind); +} + +/** + * The extractEnvironment function traverses the syntax tree from the given position downwards to collect variable and function names in scope. + * It returns an Environment object representing the current scope and its child scope unless the position is inside a function parameter list, in which case it returns null. + * + * @param iter The TreeCursor used to traverse the syntax tree + * @param pos The 0-based index position in the document for which to extract the environment + * @param doc The document text, used to extract variable and function names from the syntax nodes + * @returns The Environment object representing the current scope and its child scopes, or null if the position is inside a function parameter list + */ +const extractEnvironment = (iter: TreeCursor, pos: number, doc: string): Environment | null => { + const topEnv: Environment = { + variables: [], + functions: [], + child: null, + }; + + let currentEnv = topEnv; + do { + if (iter.node.type.name === "ParamList") { + // If the position is inside a function parameter list, + // we don't want to suggest any names + return null; + } + if (iter.node.type.name === "ForStatement") { + // Add loop variable to current environment + const target = iter.node.getChild("VariableName"); + if (target) { + currentEnv.variables.push(getNodeText(target, doc)); + } + } + if (iter.node.type.name == "FunctionDefinition" || iter.node.type.name == "LambdaExpression") { + // Add function parameters to inner environment + const params = iter.node.getChild("ParamList"); + if (params) { + for (let param = params.firstChild; param; param = param.nextSibling) { + if (param.type.name === "VariableName") { + currentEnv.variables.push(getNodeText(param, doc)); + } + } + } + continue; + } + if (iter.node.type.name !== "Body" && iter.node.type.name !== "Script") { + continue; + } + // Iterate children + for (let child = iter.node.firstChild; child; child = child.nextSibling) { + // Assignment → variable + if (child.type.name === "AssignStatement") { + const left = child.firstChild; + if ( + left && + left.type.name === "VariableName" && + (left.from != iter.node.from || left.to != iter.node.to) + ) { + currentEnv.variables.push(getNodeText(left, doc)); + } + } + + // Function + if (child.type.name === "FunctionDefinition") { + const nameNode = child.getChild("VariableName"); + if (nameNode) { + currentEnv.functions.push(getNodeText(nameNode, doc)); + } + } + } + const nextEnv = { + variables: [], + functions: [], + child: null, + }; + currentEnv.child = nextEnv; + currentEnv = nextEnv; + } while (iter.enter(pos, -1)); + + // If the node is a function name or loop variable, we don't want to include it in the autocomplete suggestions since it's not in scope at the cursor position. We check for this case by looking at the parent node of the current position - if it's a FunctionDefinition or ForStatement, we return null to indicate that no environment should be extracted. + if ( + iter.node.parent && + ["FunctionDefinition", "ForStatement"].includes(iter.node.parent.type.name) + ) { + return null; + } + return topEnv; +}; + +/** + * Checks if `sub` is a subsequence of `str`, meaning all characters of `sub` appear in `str` in the same order, but not necessarily contiguously. + * @param sub The subsequence to check + * @param str The string to check against + * @returns true if `sub` is a subsequence of `str`, false otherwise + */ +const isSubsequence = (sub: string, str: string): boolean => { + let i = 0; + for (const char of str) { + if (char === sub[i]) { + i++; + } + if (i === sub.length) { + return true; + } + } + return false; +}; + +/** + * Converts a 1-based line and column number to a 0-based index in the document string. If the line or column is out of bounds, + * it returns the closest valid index (e.g. end of document). + * + * @param doc The document text + * @param line The 1-based line number + * @param column The 1-based column number + * @returns The 0-based index in the document string corresponding to the given line and column, or the closest valid index if out of bounds + */ +const convertPosToIndex = (doc: string, line: number, column: number): number => { + let pos = 0; + while (line > 0) { + const newlineIndex = doc.indexOf("\n", pos); + if (newlineIndex === -1) { + return doc.length; + } + pos = newlineIndex + 1; + line--; + } + return pos + column; +}; + +/** + * Gets the names of variables and functions in scope at the given position in the document, + * as well as built-in functions, variables and keywords. + * @param tree The syntax tree of the document, created using Lezer + * @param doc The document text + * @param line The 1-based line number of the cursor position + * @param column The 1-based column number of the cursor position + * @param variant The variant of the language + * @returns a list of autocomplete entries, each containing the name, type (variable or function), and documentation (for built-ins) + */ +export const getNames = ( + tree: Tree, + doc: string, + line: number, + column: number, + variant: number, +): AutoCompleteEntry[] => { + const pos = convertPosToIndex(doc, line - 1, column); // Convert position to 0-based index + const node = tree.resolve(pos, -1); // Get the syntax node ending at the cursor position + if (node.type.name !== "VariableName") { + return []; + } + + const query = getNodeText(node, doc); + + let env: Environment | null = extractEnvironment(tree.cursor(), pos, doc); + if (env === null) { + return []; + } + const entries: AutoCompleteEntry[] = []; + let score = 1; // The score is used to prioritize suggestions from inner scopes over outer scopes. Built-ins will have the lowest score. + while (env) { + const symbols = [ + ...env.variables.map(v => ({ name: v, meta: CompletionItemKind.Variable, score: score })), + ...env.functions.map(f => ({ name: f, meta: CompletionItemKind.Function, score: score })), + ]; + // Filter symbols based on query, sort alphabetically, and add to entries + symbols + .filter(s => isSubsequence(query, s.name)) + .sort((a, b) => a.name.localeCompare(b.name)) + .forEach(s => entries.push(s)); + + env = env.child; + score++; + } + + getKeywords(variant) + .map(k => ({ + name: k, + meta: CompletionItemKind.Keyword, + score: score, // Keywords are given the highest score + })) + .filter(s => isSubsequence(query, s.name)) + .forEach(s => entries.push(s)); + + // TODO: Add docstrings for user-defined functions to autocomplete suggestions? + const symbols = [...miscJSON, ...mathJSON]; + if (variant >= 2) { + symbols.push(...linkedListJSON); + } + if (variant >= 3) { + symbols.push({ + name: "range", + title: "range(start, [stop], [step]) -> range", + description: + "PRIMITIVE\nUsed with the for statement to create a loop that iterates a specific number of times. If given one argument, it iterates from 0 to that number (exclusive). If given two arguments, it iterates from the first (inclusive) to the second (exclusive). If given three arguments, it iterates from the first to the second in steps of the third.", + meta: "func", + }); + symbols.push(...listJSON); + symbols.push(...pairmutatorJSON); + symbols.push(...streamJSON); + } + // TODO: add when MCE documentation is pushed into main + // if (variant >= 4) { + // symbols.push(...mceJSON); + // } + symbols + .map(v => ({ + name: v.name, + meta: isCompletionItemKind(v.meta) ? v.meta : CompletionItemKind.Variable, + docHTML: "

" + v.title + "

" + v.description + "

", + })) + .filter(s => isSubsequence(query, s.name)) + .sort((a, b) => a.name.localeCompare(b.name)) + .forEach(s => entries.push(s)); + + return entries; +}; diff --git a/src/engines/cse/interpreter.ts b/src/engines/cse/interpreter.ts index 8bf2e900..899d53db 100644 --- a/src/engines/cse/interpreter.ts +++ b/src/engines/cse/interpreter.ts @@ -1005,13 +1005,12 @@ const cmdEvaluators: CmdEvaluators = { }, [InstrType.FOR]: function ( code: string, - command: ControlItem, + instr: ForInstr, context: Context, control: Control, stash: Stash, _isPrelude: boolean, ) { - const instr = command as ForInstr; const step = stash.pop(); const end = stash.pop(); const start = stash.pop(); diff --git a/src/generate-autocomplete.mts b/src/generate-autocomplete.mts new file mode 100644 index 00000000..10abfdac --- /dev/null +++ b/src/generate-autocomplete.mts @@ -0,0 +1,108 @@ +import { CompletionItemKind } from "@sourceacademy/autocomplete"; +import { readdir, writeFile } from "node:fs/promises"; + +/** + * This script generates autocomplete data for built-in functions and variables + * by parsing raw JSDoc ASTs in `src/conductor/plugins/autocomplete/builtins`. + * It assumes that the JSDoc ASTs are already generated by running `jsdoc -X` on the built-in JavaScript files. + * + * In most circumstances, you should not need to run this script manually, + * as it is automatically invoked by the `build` script in `package.json`. + * If you want to regenerate the autocomplete data, you can run `./scripts/autocomplete.sh`, + * which will first regenerate the JSDoc ASTs and then run this script to generate the autocomplete data. + * + * It outputs JSON files containing the name, title, description, and meta information for each function and variable, + * which are then used by the autocomplete plugin. + * + * Note: This script may not have all the type information for the built-ins. Feel free to edit the Entry type and the mapping logic to include more information if needed. + */ + +type Entry = + | { + comment: string; + description: string; + params: { + type: { + names: string[]; + }; + description: string; + name: string; + }[]; + returns?: { + type: { + names: string[]; + }; + description: string; + }[]; + name: string; + longname: string; + kind: "function"; + scope: string; + } + | { + kind: "package"; + longname: string; + files: string[]; + } + | { + comment: string; + description: string; + type: { + names: string[]; + }; + name: string; + longname: string; + kind: "constant"; + scope: string; + }; + +/** + * The generation process is as follows: + * 1. Find all the JSON files in the builtins directory, which are generated by running `jsdoc -X` on the built-in JavaScript files. + * 2. For each JSON file, create a promise which + * a. Reads the JSON file and parses it into an array of Entry objects. + * b. Filters the entries to only include functions and constants, and maps them to a simplified format containing the name, title, description, and meta information. + * c. Writes the simplified entries back to the same JSON file, overwriting the original JSDoc AST. + * 3. Wait for all the promises to complete before exiting the script. + */ +(async () => { + return Promise.all( + (await readdir("src/conductor/plugins/autocomplete/builtins")) + .filter(file => file.endsWith(".json")) + .map(async file => { + const data = (await import(`./conductor/plugins/autocomplete/builtins/${file}`, { + with: { type: "json" }, + })) as { + default: Entry[]; + }; + await writeFile( + `src/conductor/plugins/autocomplete/builtins/${file}`, + JSON.stringify( + data.default + .filter(entry => entry.kind === "function" || entry.kind === "constant") + .filter(entry => entry.scope === "global") // Only include global variables and functions in autocomplete suggestions + .map(entry => + entry.kind === "function" + ? { + name: entry.name, + title: + entry.name + + "(" + + entry.params.map(p => p.name).join(", ") + + ") -> " + + (entry.returns?.[0].type.names[0] || "None"), + description: entry.description, + meta: CompletionItemKind.Function, + } + : { + name: entry.name, + title: entry.name, + description: entry.description, + meta: CompletionItemKind.Variable, + }, + ), + ), + ); + }), + ); +})(); diff --git a/src/tests/autocomplete.test.ts b/src/tests/autocomplete.test.ts new file mode 100644 index 00000000..ae5c95cc --- /dev/null +++ b/src/tests/autocomplete.test.ts @@ -0,0 +1,182 @@ +import { parser } from "@lezer/python"; +import { CompletionItemKind } from "@sourceacademy/autocomplete"; +import { getNames } from "../../src/conductor/plugins/autocomplete/resolver"; +const testContains = ( + code: string, + expected: { name: string; meta: CompletionItemKind }, + line: number, + column: number, + variant: number, +) => { + const tree = parser.parse(code); + const suggestions = getNames(tree, code, line, column, variant); + expect(suggestions).toContainEqual(expect.objectContaining(expected)); +}; + +const testNotContains = ( + code: string, + expected: { name: string; meta: CompletionItemKind }, + line: number, + column: number, + variant: number, +) => { + const tree = parser.parse(code); + const suggestions = getNames(tree, code, line, column, variant); + expect(suggestions).not.toContainEqual(expect.objectContaining(expected)); +}; +describe("Chapter 1 Autocomplete", () => { + test("should suggest built-in functions", () => { + testContains("le", { name: "len", meta: CompletionItemKind.Function }, 1, 2, 1); + }); + test("should not suggest built-ins when not a subsequence", () => { + testNotContains("el", { name: "len", meta: CompletionItemKind.Function }, 1, 2, 1); + }); + test("should not suggest Chapter 3 keywords", () => { + testNotContains("wh", { name: "while", meta: CompletionItemKind.Keyword }, 1, 2, 1); + testNotContains("fo", { name: "for", meta: CompletionItemKind.Keyword }, 1, 2, 1); + testNotContains("br", { name: "break", meta: CompletionItemKind.Keyword }, 1, 2, 1); + testNotContains("co", { name: "continue", meta: CompletionItemKind.Keyword }, 1, 2, 1); + testNotContains("i", { name: "in", meta: CompletionItemKind.Keyword }, 1, 1, 1); + }); + test("should suggest keywords", () => { + testContains("de", { name: "def", meta: CompletionItemKind.Keyword }, 1, 2, 1); + }); + test("can handle no suggestions", () => { + const tree = parser.parse("x = 10\nx."); + const suggestions = getNames(tree, "x = 10\nx.", 2, 3, 1); + expect(suggestions).toEqual([]); + }); + test("should suggest variables in scope", () => { + testContains( + "x = 10\ny = x + 5\nzab = y * 2\nza", + { name: "zab", meta: CompletionItemKind.Variable }, + 4, + 2, + 1, + ); + }); + test("can handle layers of scope", () => { + testContains( + "x = 10\ndef foo():\n y = x + 5\n def bar():\n zab = y * 2\n za", + { name: "zab", meta: CompletionItemKind.Variable }, + 6, + 10, + 1, + ); + testContains( + "x = 10\ndef foo():\n y = x + 5\n def bar():\n zab = y * 2\n y", + { name: "y", meta: CompletionItemKind.Variable }, + 6, + 9, + 1, + ); + testContains( + "x = 10\ndef foo():\n y = x + 5\n def bar():\n zab = y * 2\n x", + { name: "x", meta: CompletionItemKind.Variable }, + 6, + 9, + 1, + ); + + testNotContains( + "x = 10\ndef foo():\n y = x + 5\n def bar():\n zab = y * 2\nza", + { name: "zab", meta: CompletionItemKind.Variable }, + 6, + 2, + 1, + ); + testNotContains( + "x = 10\ndef foo():\n y = x + 5\n def bar():\n zab = y * 2\ny", + { name: "y", meta: CompletionItemKind.Variable }, + 6, + 1, + 1, + ); + testContains( + "x = 10\ndef foo(x):\n y = x + 5\n def bar():\n zab = y * 2\nx", + { name: "x", meta: CompletionItemKind.Variable }, + 6, + 1, + 1, + ); + }); + test("does not suggest name during function definition", () => { + testNotContains("foo = 3\ndef f", { name: "f", meta: CompletionItemKind.Function }, 2, 5, 1); + testNotContains("foo = 3\ndef f", { name: "foo", meta: CompletionItemKind.Variable }, 2, 5, 1); + testNotContains( + "bar = 3\ndef f(b", + { name: "bar", meta: CompletionItemKind.Variable }, + 2, + 7, + 1, + ); + }); + test("suggests name during function call", () => { + testContains( + "foo = 3\ndef f():\n pass\nf(fo", + { name: "foo", meta: CompletionItemKind.Variable }, + 4, + 4, + 1, + ); + }); +}); + +describe("Chapter 3 Autocomplete", () => { + test("while loops internals should not be visible", () => { + testNotContains( + "x = 10\nwhile x > 0:\n y = x + 5\n x -= 1\n zab = y * 2\nza", + { name: "y", meta: CompletionItemKind.Variable }, + 6, + 2, + 3, + ); + testNotContains( + "x = 10\nwhile x > 0:\n y = x + 5\n x -= 1\n zab = y * 2\ny", + { name: "zab", meta: CompletionItemKind.Variable }, + 6, + 1, + 3, + ); + }); + test("for loops should have the loop variable visible inside the loop", () => { + testContains( + "x = 10\nfor i in range(x):\n y = i + 5\n zab = y * 2\n i", + { name: "i", meta: CompletionItemKind.Variable }, + 5, + 5, + 3, + ); + }); + + test("for loop internals should not be visible", () => { + testNotContains( + "x = 10\nfor i in range(x):\n y = i + 5\n zab = y * 2\nza", + { name: "zab", meta: CompletionItemKind.Variable }, + 5, + 2, + 3, + ); + testNotContains( + "x = 10\nfor i in range(x):\n y = i + 5\n zab = y * 2\ny", + { name: "y", meta: CompletionItemKind.Variable }, + 5, + 1, + 3, + ); + testNotContains( + "x = 10\nfor i in range(x):\n y = i + 5\n zab = y * 2\ni", + { name: "i", meta: CompletionItemKind.Variable }, + 5, + 1, + 3, + ); + }); + test("should suggest Chapter 3 keywords", () => { + testContains("wh", { name: "while", meta: CompletionItemKind.Keyword }, 1, 2, 3); + testContains("fo", { name: "for", meta: CompletionItemKind.Keyword }, 1, 2, 3); + testContains("br", { name: "break", meta: CompletionItemKind.Keyword }, 1, 2, 3); + testContains("co", { name: "continue", meta: CompletionItemKind.Keyword }, 1, 2, 3); + testContains("i", { name: "in", meta: CompletionItemKind.Keyword }, 1, 1, 3); + }); +}); diff --git a/yarn.lock b/yarn.lock index bb2ef638..2c06aced 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1259,6 +1259,42 @@ __metadata: languageName: node linkType: hard +"@lezer/common@npm:^1.0.0, @lezer/common@npm:^1.2.0, @lezer/common@npm:^1.3.0": + version: 1.5.1 + resolution: "@lezer/common@npm:1.5.1" + checksum: 10c0/49baefdfc6f2244ad4f7d4a318149729fbecfd634fe1f7769883b5098ab9b35429140851e524c3a97614594004d8a3ad08fdd91221a63438be8c31ff2431fb54 + languageName: node + linkType: hard + +"@lezer/highlight@npm:^1.0.0": + version: 1.2.3 + resolution: "@lezer/highlight@npm:1.2.3" + dependencies: + "@lezer/common": "npm:^1.3.0" + checksum: 10c0/3bcb4fce7a1a45b5973895d7cb2be47970a0098700f2a0970aef9878ffd37f540285a2d7388ec1f524726ec90cc5196b5701bbb9764b7e7300786d772b7d2ce2 + languageName: node + linkType: hard + +"@lezer/lr@npm:^1.0.0": + version: 1.4.8 + resolution: "@lezer/lr@npm:1.4.8" + dependencies: + "@lezer/common": "npm:^1.0.0" + checksum: 10c0/8bd2228a316a5ef8da01908e3e22aca95fa9695211ffe56f3e8be756b37d0810d5aa91fbbdd274b198a343051d8637e130e26f51161161f089244af242b653c9 + languageName: node + linkType: hard + +"@lezer/python@npm:^1.1.18": + version: 1.1.18 + resolution: "@lezer/python@npm:1.1.18" + dependencies: + "@lezer/common": "npm:^1.2.0" + "@lezer/highlight": "npm:^1.0.0" + "@lezer/lr": "npm:^1.0.0" + checksum: 10c0/8d984729e887808c75800f18ed54560adfd4e67094b301a1666bdcd49e8987ab45f04c515563a92dfb1377d4a04dcf6616adc50a75285afe9ab53ab90f659bd5 + languageName: node + linkType: hard + "@npmcli/agent@npm:^4.0.0": version: 4.0.0 resolution: "@npmcli/agent@npm:4.0.0" @@ -1636,6 +1672,15 @@ __metadata: languageName: node linkType: hard +"@sourceacademy/autocomplete@github:source-academy/autocomplete#0.0.1": + version: 0.0.1 + resolution: "@sourceacademy/autocomplete@https://github.com/source-academy/autocomplete.git#commit=db3ba73eedcef82eff6b4031e423804b67a2bb6e" + peerDependencies: + "@sourceacademy/conductor": ">=0.3.0" + checksum: 10c0/0c836d923d5658071882b4125445237f99cbef97f2f0b2834f0b244587a5dabd44944e4c6cda766ad3458c6646a8302bf7354a513b7486ec174fb46809f4a08b + languageName: node + linkType: hard + "@sourceacademy/conductor@npm:^0.3.0": version: 0.3.0 resolution: "@sourceacademy/conductor@npm:0.3.0" @@ -4760,6 +4805,7 @@ __metadata: resolution: "py-slang@workspace:." dependencies: "@inquirer/prompts": "npm:^8.3.2" + "@lezer/python": "npm:^1.1.18" "@rollup/plugin-commonjs": "npm:^28.0.3" "@rollup/plugin-json": "npm:^6.1.0" "@rollup/plugin-node-resolve": "npm:^16.0.1" @@ -4767,6 +4813,7 @@ __metadata: "@rollup/plugin-terser": "npm:^1.0.0" "@rollup/plugin-typescript": "npm:^12.1.2" "@rollup/plugin-wasm": "npm:^6.2.2" + "@sourceacademy/autocomplete": "github:source-academy/autocomplete#0.0.1" "@sourceacademy/conductor": "npm:^0.3.0" "@sourceacademy/wasm-util": "npm:^1.0.6" "@types/fast-levenshtein": "npm:^0.0.4"