Skip to content

Commit 0bb8eac

Browse files
authored
Jedi Autocomplete (#3114)
1 parent 050c6c5 commit 0bb8eac

13 files changed

Lines changed: 686 additions & 191 deletions

File tree

docker/server-jetty/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ def targetArch = Architecture.targetArchitecture(project)
1010

1111
def baseMapAmd64 = [
1212
'server-base': 'server-jetty',
13+
'all-ai-base': 'server-all-ai-jetty',
1314
]
1415

1516
// Only the server image is supported on arm64
Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
absl-py==1.3.0
2+
astunparse==1.6.3
3+
cachetools==5.2.0
4+
certifi==2022.9.24
5+
charset-normalizer==2.1.1
6+
click==8.1.3
7+
deephaven-plugin==0.3.0
8+
flatbuffers==2.0.7
9+
gast==0.4.0
10+
google-auth==2.14.1
11+
google-auth-oauthlib==0.4.6
12+
google-pasta==0.2.0
13+
grpcio==1.51.1
14+
h5py==3.7.0
15+
idna==3.4
16+
importlib-metadata==5.1.0
17+
java-utilities==0.2.0
18+
jedi==0.18.2
19+
joblib==1.2.0
20+
jpy==0.13.0
21+
keras==2.7.0
22+
Keras-Preprocessing==1.1.2
23+
libclang==14.0.6
24+
llvmlite==0.39.1
25+
Markdown==3.4.1
26+
MarkupSafe==2.1.1
27+
nltk==3.7
28+
numba==0.56.4
29+
numpy==1.21.6
30+
nvidia-cublas-cu11==11.10.3.66
31+
nvidia-cuda-nvrtc-cu11==11.7.99
32+
nvidia-cuda-runtime-cu11==11.7.99
33+
nvidia-cudnn-cu11==8.5.0.96
34+
oauthlib==3.2.2
35+
opt-einsum==3.3.0
36+
pandas==1.3.5
37+
parso==0.8.3
38+
protobuf==3.19.6
39+
pyasn1==0.4.8
40+
pyasn1-modules==0.2.8
41+
python-dateutil==2.8.2
42+
pytz==2022.6
43+
regex==2022.10.31
44+
requests==2.28.1
45+
requests-oauthlib==1.3.1
46+
rsa==4.9
47+
scikit-learn==1.0.2
48+
scipy==1.7.3
49+
six==1.16.0
50+
tensorboard==2.11.0
51+
tensorboard-data-server==0.6.1
52+
tensorboard-plugin-wit==1.8.1
53+
tensorflow==2.7.4
54+
tensorflow-estimator==2.7.0
55+
tensorflow-io-gcs-filesystem==0.28.0
56+
termcolor==2.1.1
57+
threadpoolctl==3.1.0
58+
torch==1.13.0
59+
tqdm==4.64.1
60+
typing_extensions==4.4.0
61+
urllib3==1.26.13
62+
Werkzeug==2.2.2
63+
wrapt==1.14.1
64+
zipp==3.11.0

open-api/lang-parser/src/main/java/io/deephaven/lang/parse/CompletionParser.java

Lines changed: 43 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,36 @@ public class CompletionParser implements Closeable {
2424
private static final Logger LOGGER = LoggerFactory.getLogger(CompletionParser.class);
2525
private final Map<String, PendingParse> docs = new ConcurrentHashMap<>();
2626

27+
public static String updateDocumentChanges(final String uri, final int version, String document,
28+
final List<ChangeDocumentRequest.TextDocumentContentChangeEvent> changes) {
29+
for (ChangeDocumentRequest.TextDocumentContentChangeEventOrBuilder change : changes) {
30+
DocumentRange range = change.getRange();
31+
int length = change.getRangeLength();
32+
33+
int offset = LspTools.getOffsetFromPosition(document, range.getStart());
34+
if (offset < 0) {
35+
if (LOGGER.isWarnEnabled()) {
36+
LOGGER.warn()
37+
.append("Invalid change in document ")
38+
.append(uri)
39+
.append("[")
40+
.append(version)
41+
.append("] @")
42+
.append(range.getStart().getLine())
43+
.append(":")
44+
.append(range.getStart().getCharacter())
45+
.endl();
46+
}
47+
return null;
48+
}
49+
50+
String prefix = offset > 0 && offset <= document.length() ? document.substring(0, offset) : "";
51+
String suffix = offset + length < document.length() ? document.substring(offset + length) : "";
52+
document = prefix + change.getText() + suffix;
53+
}
54+
return document;
55+
}
56+
2757
public ParsedDocument parse(String document) throws ParseException {
2858
Chunker chunker = new Chunker(document);
2959
final ChunkerDocument doc = chunker.Document();
@@ -49,7 +79,7 @@ private PendingParse startParse(String uri) {
4979
return docs.computeIfAbsent(uri, k -> new PendingParse(uri));
5080
}
5181

52-
public void update(final String uri, final String version,
82+
public void update(final String uri, final int version,
5383
final List<ChangeDocumentRequest.TextDocumentContentChangeEvent> changes) {
5484
if (LOGGER.isTraceEnabled()) {
5585
LOGGER.trace()
@@ -74,32 +104,11 @@ public void update(final String uri, final String version,
74104
forceParse = true;
75105
}
76106
String document = doc.getText();
77-
for (ChangeDocumentRequest.TextDocumentContentChangeEventOrBuilder change : changes) {
78-
DocumentRange range = change.getRange();
79-
int length = change.getRangeLength();
80-
81-
int offset = LspTools.getOffsetFromPosition(document, range.getStart());
82-
if (offset < 0) {
83-
if (LOGGER.isWarnEnabled()) {
84-
LOGGER.warn()
85-
.append("Invalid change in document ")
86-
.append(uri)
87-
.append("[")
88-
.append(version)
89-
.append("] @")
90-
.append(range.getStart().getLine())
91-
.append(":")
92-
.append(range.getStart().getCharacter())
93-
.endl();
94-
}
95-
return;
96-
}
97-
98-
String prefix = offset > 0 && offset <= document.length() ? document.substring(0, offset) : "";
99-
String suffix = offset + length < document.length() ? document.substring(offset + length) : "";
100-
document = prefix + change.getText() + suffix;
107+
document = updateDocumentChanges(uri, version, document, changes);
108+
if (document == null) {
109+
return;
101110
}
102-
doc.requestParse(version, document, forceParse);
111+
doc.requestParse(Integer.toString(version), document, forceParse);
103112
if (LOGGER.isTraceEnabled()) {
104113
LOGGER.trace()
105114
.append("Finished updating ")
@@ -118,6 +127,14 @@ public void remove(String uri) {
118127
}
119128
}
120129

130+
public String getText(String uri) {
131+
final PendingParse doc = docs.get(uri);
132+
if (doc == null) {
133+
throw new IllegalStateException("Unable to find parsed document " + uri);
134+
}
135+
return doc.getText();
136+
}
137+
121138
public ParsedDocument finish(String uri) {
122139
final PendingParse doc = docs.get(uri);
123140
if (doc == null) {

open-api/lang-tools/src/main/java/io/deephaven/lang/completion/ChunkerCompleter.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,7 @@ private TextEdit.Builder extendEnd(final CompletionItem.Builder item, final Posi
354354
}
355355

356356

357-
private String sortable(int i) {
357+
public static String sortable(int i) {
358358
StringBuilder res = new StringBuilder(Integer.toString(i, 36));
359359
while (res.length() < 5) {
360360
res.insert(0, "0");

open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerCompletionHandlerTest.groovy

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,8 +160,8 @@ b = 2
160160
c = 3
161161
"""
162162
String src2 = "t = "
163-
p.update(uri, "0", [ makeChange(0, 0, src1) ])
164-
p.update(uri, "1", [ makeChange(3, 0, src2) ])
163+
p.update(uri, 0, [ makeChange(0, 0, src1) ])
164+
p.update(uri, 1, [ makeChange(3, 0, src2) ])
165165
doc = p.finish(uri)
166166

167167
VariableProvider variables = Mock(VariableProvider) {
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#
2+
# Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending
3+
#
4+
5+
""" This module allows the user to configure if and how we use jedi to perform autocompletion.
6+
See https://github.com/davidhalter/jedi for information on jedi.
7+
8+
# To disable autocompletion
9+
from deephaven.completer import jedi_settings
10+
jedi_settings.mode = 'off'
11+
12+
Valid options for completer_mode are one of: [off, safe, strong].
13+
off: do not use any autocomplete
14+
safe mode: uses static analysis of source files. Can't execute any code.
15+
strong mode: looks in your globals() for answers to autocomplete and analyzes your runtime python objects
16+
later, we may add slow mode, which uses both static and interpreted completion modes.
17+
"""
18+
19+
from deephaven.completer._completer import Completer
20+
from jedi import preload_module, Interpreter
21+
22+
jedi_settings = Completer()
23+
# warm jedi up a little. We could probably off-thread this.
24+
preload_module('deephaven')
25+
Interpreter('', []).complete(1, 0)
Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
# only python 3.8 needs this, but it must be the first expression in the file, so we can't predicate it
2+
from __future__ import annotations
3+
from enum import Enum
4+
from typing import Any
5+
from jedi import Interpreter, Script
6+
7+
8+
class CompleterMode(Enum):
9+
off = 'off'
10+
safe = 'safe'
11+
strong = 'strong'
12+
13+
def __str__(self) -> str:
14+
return self.value
15+
16+
17+
class Completer(object):
18+
19+
def __init__(self):
20+
self._docs = {}
21+
self._versions = {}
22+
# we will replace this w/ top-level globals() when we open the document
23+
self.__scope = globals()
24+
# might want to make this a {uri: []} instead of []
25+
self.pending = []
26+
try:
27+
import jedi
28+
self.__can_jedi = True
29+
self.mode = CompleterMode.strong
30+
except ImportError:
31+
self.__can_jedi = False
32+
self.mode = CompleterMode.off
33+
34+
@property
35+
def mode(self) -> CompleterMode:
36+
return self.__mode
37+
38+
@mode.setter
39+
def mode(self, mode) -> None:
40+
if type(mode) == 'str':
41+
mode = CompleterMode[mode]
42+
self.__mode = mode
43+
44+
def open_doc(self, text: str, uri: str, version: int) -> None:
45+
self._docs[uri] = text
46+
self._versions[uri] = version
47+
48+
def get_doc(self, uri: str) -> str:
49+
return self._docs[uri]
50+
51+
def update_doc(self, text: str, uri: str, version: int) -> None:
52+
self._docs[uri] = text
53+
self._versions[uri] = version
54+
# any pending completions should stop running now. We use a list of Event to signal any running threads to stop
55+
for pending in self.pending:
56+
pending.set()
57+
58+
def close_doc(self, uri: str) -> None:
59+
del self._docs[uri]
60+
del self._versions[uri]
61+
for pending in self.pending:
62+
pending.set()
63+
64+
def is_enabled(self) -> bool:
65+
return self.__mode != CompleterMode.off
66+
67+
def can_jedi(self) -> bool:
68+
return self.__can_jedi
69+
70+
def set_scope(self, scope: dict) -> None:
71+
self.__scope = scope
72+
73+
def do_completion(self, uri: str, version: int, line: int, col: int) -> list[list[Any]]:
74+
if not self._versions[uri] == version:
75+
# if you aren't the newest completion, you get nothing, quickly
76+
return []
77+
78+
# run jedi
79+
txt = self.get_doc(uri)
80+
# The Script completer is static analysis only, so we should actually be feeding it a whole document at once.
81+
82+
completer = Script if self.__mode == CompleterMode.safe else Interpreter
83+
84+
completions = completer(txt, [self.__scope]).complete(line, col)
85+
# for now, a simple sorting based on number of preceding _
86+
# we may want to apply additional sorting to each list before combining
87+
results: list = []
88+
results_: list = []
89+
results__: list = []
90+
for complete in completions:
91+
# keep checking the latest version as we run, so updated doc can cancel us
92+
if not self._versions[uri] == version:
93+
return []
94+
result: list = self.to_result(complete, col)
95+
if result[0].startswith('__'):
96+
results__.append(result)
97+
elif result[0].startswith('_'):
98+
results_.append(result)
99+
else:
100+
results.append(result)
101+
102+
# put the results together in a better-than-nothing sorting
103+
return results + results_ + results__
104+
105+
@staticmethod
106+
def to_result(complete: Any, col: int) -> list[Any]:
107+
name: str = complete.name
108+
prefix_length: int = complete.get_completion_prefix_length()
109+
start: int = col - prefix_length
110+
# all java needs to build a grpc response is completion text (name) and where the completion should start
111+
return [name, start]

py/server/deephaven/config/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,6 @@ def get_server_timezone() -> TimeZone:
1919
for tz in TimeZone:
2020
if j_timezone == tz.value.getTimeZone():
2121
return tz
22-
raise NotImplementedError("can't find the time zone in the TImeZone Enum.")
22+
raise NotImplementedError("can't find the time zone in the TimeZone Enum.")
2323
except Exception as e:
2424
raise DHError(e, message=f"failed to find a recognized time zone") from e

py/server/setup.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ def normalize_version(version):
6262
# TODO(deephaven-core#3082): Remove numba dependency workarounds
6363
'numba; python_version < "3.11"',
6464
],
65+
extras_require={
66+
"autocomplete": ["jedi==0.18.2"],
67+
},
6568
entry_points={
6669
'deephaven.plugin': ['registration_cls = deephaven.pandasplugin:PandasPluginRegistration']
6770
}

server/jetty-app/build.gradle

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,10 @@ if (hasProperty('debug')) {
6565
extraJvmArgs += ['-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005']
6666
}
6767

68+
if (hasProperty('debugAutocomplete')) {
69+
extraJvmArgs += ['-Ddeephaven.console.autocomplete.quiet=false']
70+
}
71+
6872
if (hasProperty('gcApplication')) {
6973
extraJvmArgs += ['-Dio.deephaven.app.GcApplication.enabled=true']
7074
}

0 commit comments

Comments
 (0)