Skip to content

Commit 2845e01

Browse files
headiusheadius
committed
Separate native and wasm parsers
Both forms are still in the same provider, but separated into two classes. The ParserProvider logic for JRuby may need some tweaks to support two separate SPIs so this is a first step to isolating them. The native parser will be returned if: * A dynamic library file provided to initialize exists, and * the wasm parser has not been specifically requested. The wasm parser will be returned if: * It was specifically requested, or * a dynamic library that exists has not been provided to initialize. In the future this selection should happen closer to the JRuby level so we can use either or both more easily.
1 parent 421936a commit 2845e01

4 files changed

Lines changed: 150 additions & 59 deletions

File tree

‎src/main/java/org/jruby/prism/ParserProviderPrism.java‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,43 @@
11
package org.jruby.prism;
22

3+
import jnr.ffi.LibraryLoader;
34
import org.jruby.Ruby;
45
import org.jruby.ir.builder.IRBuilderFactory;
56
import org.jruby.parser.Parser;
7+
import org.jruby.parser.ParserManager;
68
import org.jruby.parser.ParserProvider;
7-
import org.jruby.prism.parser.ParserPrism;
8-
import org.jruby.prism.parser.ParserBindingPrism;
99
import org.jruby.prism.builder.IRBuilderFactoryPrism;
10+
import org.jruby.prism.parser.ParserBindingPrism;
11+
import org.jruby.prism.parser.ParserPrismNative;
12+
import org.jruby.prism.parser.ParserPrismWasm;
1013

11-
import jnr.ffi.LibraryLoader;
14+
import java.io.File;
1215

1316
public class ParserProviderPrism implements ParserProvider {
1417
private static ParserBindingPrism prismLibrary;
1518

1619
public void initialize(String path) {
17-
if (prismLibrary != null) {
18-
System.out.println("Prism already initialized");
19-
return;
20+
if (new File(path).exists()) {
21+
if (prismLibrary != null) {
22+
System.out.println("Prism already initialized");
23+
return;
24+
}
25+
prismLibrary = LibraryLoader.create(ParserBindingPrism.class).load(path);
26+
// We do something extra here as a side-effect which is how we get an UnsatisfiedLinkError
27+
// If the library didn't in fact find the .so or has other loading problems.
28+
ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary));
29+
} else {
30+
prismLibrary = null;
2031
}
21-
prismLibrary = LibraryLoader.create(ParserBindingPrism.class).load(path);
22-
// We do something extra here as a side-effect which is how we get an UnsatisfiedLinkError
23-
// If the library didn't in fact find the .so or has other loading problems.
24-
ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary));
2532
}
2633

2734
public Parser getParser(Ruby runtime) {
28-
return new ParserPrism(runtime, prismLibrary);
35+
if (ParserManager.PARSER_WASM || prismLibrary == null) {
36+
// uninitialized dynamic lib or wasm requested
37+
return new ParserPrismWasm(runtime);
38+
}
39+
40+
return new ParserPrismNative(runtime, prismLibrary);
2941
}
3042

3143
public IRBuilderFactory getBuilderFactory() {

src/main/java/org/jruby/prism/parser/ParserPrism.java renamed to src/main/java/org/jruby/prism/parser/ParserPrismBase.java

Lines changed: 22 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
import org.jruby.ext.coverage.CoverageData;
1212
import org.jruby.management.ParserStats;
1313
import org.jruby.parser.Parser;
14-
import org.jruby.parser.ParserManager;
1514
import org.jruby.parser.ParserType;
1615
import org.jruby.parser.StaticScope;
1716
import org.jruby.runtime.DynamicScope;
@@ -21,10 +20,16 @@
2120
import org.jruby.util.ByteList;
2221
import org.jruby.util.CommonByteLists;
2322
import org.jruby.util.io.ChannelHelper;
24-
import org.prism.Nodes;
25-
import org.prism.Nodes.*;
23+
import org.prism.Nodes.ArgumentsNode;
24+
import org.prism.Nodes.CallNode;
25+
import org.prism.Nodes.CallNodeFlags;
26+
import org.prism.Nodes.GlobalVariableReadNode;
27+
import org.prism.Nodes.GlobalVariableWriteNode;
28+
import org.prism.Nodes.Node;
29+
import org.prism.Nodes.ProgramNode;
30+
import org.prism.Nodes.StatementsNode;
31+
import org.prism.Nodes.WhileNode;
2632
import org.prism.ParsingOptions;
27-
import org.prism.Prism;
2833

2934
import java.io.ByteArrayInputStream;
3035
import java.io.DataInputStream;
@@ -39,16 +44,15 @@
3944
import static org.jruby.parser.ParserType.EVAL;
4045
import static org.jruby.parser.ParserType.MAIN;
4146

42-
public class ParserPrism extends Parser {
43-
private boolean parserTiming = org.jruby.util.cli.Options.PARSER_SUMMARY.load();
47+
public abstract class ParserPrismBase extends Parser {
48+
protected boolean parserTiming = org.jruby.util.cli.Options.PARSER_SUMMARY.load();
4449

45-
private final ParserBindingPrism prismLibrary;
46-
47-
public ParserPrism(Ruby runtime, ParserBindingPrism prismLibrary) {
50+
public ParserPrismBase(Ruby runtime) {
4851
super(runtime);
49-
this.prismLibrary = prismLibrary;
5052
}
5153

54+
protected abstract byte[] parse(byte[] source, int sourceLength, byte[] metadata);
55+
5256
@Override
5357
public ParseResult parse(String fileName, int lineNumber, ByteList content, DynamicScope existingScope, ParserType type) {
5458
int sourceLength = content.realSize();
@@ -113,10 +117,10 @@ private ParseResult parseInternal(String fileName, DynamicScope blockScope, byte
113117
coverageMode = runtime.getCoverageData().getMode();
114118
}
115119

116-
ParseResultPrism result = new ParseResultPrism(fileName, source, (Nodes.ProgramNode) res.value, res.source, encoding, coverageMode);
120+
ParseResultPrism result = new ParseResultPrism(fileName, source, (ProgramNode) res.value, res.source, encoding, coverageMode);
117121
if (blockScope != null) {
118122
if (type == MAIN) { // update TOPLEVEL_BINDNG
119-
RubySymbol[] locals = ((Nodes.ProgramNode) result.getAST()).locals;
123+
RubySymbol[] locals = ((ProgramNode) result.getAST()).locals;
120124
for (int i = 0; i < locals.length; i++) {
121125
blockScope.getStaticScope().addVariableThisScope(locals[i].idString());
122126
}
@@ -174,36 +178,6 @@ private byte[] loadFully(String fileName, InputStream in) {
174178
}
175179
}
176180

177-
178-
private byte[] parse(byte[] source, int sourceLength, byte[] metadata) {
179-
if (ParserManager.PARSER_WASM) return parseChicory(source, sourceLength, metadata);
180-
181-
long time = 0;
182-
if (parserTiming) time = System.nanoTime();
183-
184-
ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary));
185-
prismLibrary.pm_buffer_init(buffer);
186-
prismLibrary.pm_serialize_parse(buffer, source, sourceLength, metadata);
187-
if (parserTiming) {
188-
ParserStats stats = runtime.getParserManager().getParserStats();
189-
190-
stats.addPrismTimeCParseSerialize(System.nanoTime() - time);
191-
}
192-
193-
int length = buffer.length.intValue();
194-
byte[] src = new byte[length];
195-
buffer.value.get().get(0, src, 0, length);
196-
197-
return src;
198-
}
199-
200-
201-
private byte[] parseChicory(byte[] source, int sourceLength, byte[] metadata) {
202-
try (Prism prism = new Prism()) {
203-
return prism.serialize(metadata, source, sourceLength);
204-
}
205-
}
206-
207181
// lineNumber (0-indexed)
208182
private byte[] generateMetadata(String fileName, int lineNumber, Encoding encoding, DynamicScope scope, ParserType type) {
209183
ByteList metadata = new ByteList();
@@ -327,23 +301,23 @@ public IRubyObject getLineStub(ThreadContext context, ParseResult arg, int lineC
327301
@Override
328302
public ParseResult addGetsLoop(Ruby runtime, ParseResult result, boolean printing, boolean processLineEndings, boolean split) {
329303
var context = runtime.getCurrentContext();
330-
List<Nodes.Node> newBody = new ArrayList<>();
304+
List<Node> newBody = new ArrayList<>();
331305

332306
if (processLineEndings) {
333-
newBody.add(new Nodes.GlobalVariableWriteNode(-1, 0, 0, asSymbol(context, CommonByteLists.DOLLAR_BACKSLASH),
307+
newBody.add(new GlobalVariableWriteNode(-1, 0, 0, asSymbol(context, CommonByteLists.DOLLAR_BACKSLASH),
334308
new GlobalVariableReadNode(-1, 0, 0, asSymbol(context, CommonByteLists.DOLLAR_SLASH))));
335309
}
336310

337-
Nodes.GlobalVariableReadNode dollarUnderscore = new GlobalVariableReadNode(-1, 0, 0, asSymbol(context, DOLLAR_UNDERSCORE));
311+
GlobalVariableReadNode dollarUnderscore = new GlobalVariableReadNode(-1, 0, 0, asSymbol(context, DOLLAR_UNDERSCORE));
338312

339-
List<Nodes.Node> whileBody = new ArrayList<>();
313+
List<Node> whileBody = new ArrayList<>();
340314

341315
if (processLineEndings) {
342316
whileBody.add(new CallNode(-1, 0, 0, (short) 0, dollarUnderscore, asSymbol(context, "chomp!"), null, null));
343317
}
344318
if (split) {
345319
whileBody.add(new GlobalVariableWriteNode(-1, 0, 0, asSymbol(context, "$F"),
346-
new Nodes.CallNode(-1, 0, 0, (short) 0, dollarUnderscore, asSymbol(context, "split"), null, null)));
320+
new CallNode(-1, 0, 0, (short) 0, dollarUnderscore, asSymbol(context, "split"), null, null)));
347321
}
348322

349323
StatementsNode stmts = ((ProgramNode) result.getAST()).statements;
@@ -362,7 +336,7 @@ public ParseResult addGetsLoop(Ruby runtime, ParseResult result, boolean printin
362336

363337
nodes = new Node[newBody.size()];
364338
newBody.toArray(nodes);
365-
Nodes.ProgramNode newRoot = new Nodes.ProgramNode(-1, 0, 0, new RubySymbol[] {}, new StatementsNode(-1, 0, 0, nodes));
339+
ProgramNode newRoot = new ProgramNode(-1, 0, 0, new RubySymbol[] {}, new StatementsNode(-1, 0, 0, nodes));
366340

367341
((ParseResultPrism) result).setRoot(newRoot);
368342

‎src/main/java/org/jruby/prism/parser/ParserPrismNative.java‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
package org.jruby.prism.parser;
2+
3+
import org.jcodings.Encoding;
4+
import org.jcodings.specific.ISO8859_1Encoding;
5+
import org.jruby.ParseResult;
6+
import org.jruby.Ruby;
7+
import org.jruby.RubyArray;
8+
import org.jruby.RubyIO;
9+
import org.jruby.RubyInstanceConfig;
10+
import org.jruby.RubySymbol;
11+
import org.jruby.ext.coverage.CoverageData;
12+
import org.jruby.management.ParserStats;
13+
import org.jruby.parser.Parser;
14+
import org.jruby.parser.ParserManager;
15+
import org.jruby.parser.ParserType;
16+
import org.jruby.parser.StaticScope;
17+
import org.jruby.runtime.DynamicScope;
18+
import org.jruby.runtime.ThreadContext;
19+
import org.jruby.runtime.builtin.IRubyObject;
20+
import org.jruby.runtime.load.LoadServiceResourceInputStream;
21+
import org.jruby.util.ByteList;
22+
import org.jruby.util.CommonByteLists;
23+
import org.jruby.util.io.ChannelHelper;
24+
import org.prism.Nodes;
25+
import org.prism.Nodes.*;
26+
import org.prism.ParsingOptions;
27+
import org.prism.Prism;
28+
29+
import java.io.ByteArrayInputStream;
30+
import java.io.DataInputStream;
31+
import java.io.IOException;
32+
import java.io.InputStream;
33+
import java.util.ArrayList;
34+
import java.util.Arrays;
35+
import java.util.List;
36+
37+
import static org.jruby.api.Convert.asSymbol;
38+
import static org.jruby.lexer.LexingCommon.DOLLAR_UNDERSCORE;
39+
import static org.jruby.parser.ParserType.EVAL;
40+
import static org.jruby.parser.ParserType.MAIN;
41+
42+
public class ParserPrismNative extends ParserPrismBase {
43+
private final ParserBindingPrism prismLibrary;
44+
45+
public ParserPrismNative(Ruby runtime, ParserBindingPrism prismLibrary) {
46+
super(runtime);
47+
this.prismLibrary = prismLibrary;
48+
}
49+
50+
protected byte[] parse(byte[] source, int sourceLength, byte[] metadata) {
51+
long time = 0;
52+
if (parserTiming) time = System.nanoTime();
53+
54+
ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary));
55+
prismLibrary.pm_buffer_init(buffer);
56+
prismLibrary.pm_serialize_parse(buffer, source, sourceLength, metadata);
57+
if (parserTiming) {
58+
ParserStats stats = runtime.getParserManager().getParserStats();
59+
60+
stats.addPrismTimeCParseSerialize(System.nanoTime() - time);
61+
}
62+
63+
int length = buffer.length.intValue();
64+
byte[] src = new byte[length];
65+
buffer.value.get().get(0, src, 0, length);
66+
67+
return src;
68+
}
69+
}

‎src/main/java/org/jruby/prism/parser/ParserPrismWasm.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package org.jruby.prism.parser;
2+
3+
import org.jcodings.Encoding;
4+
import org.jcodings.specific.ISO8859_1Encoding;
5+
import org.jruby.*;
6+
import org.jruby.parser.Parser;
7+
import org.jruby.parser.ParserType;
8+
import org.jruby.parser.StaticScope;
9+
import org.jruby.runtime.DynamicScope;
10+
import org.jruby.runtime.ThreadContext;
11+
import org.jruby.runtime.builtin.IRubyObject;
12+
import org.jruby.util.ByteList;
13+
import org.jruby.util.CommonByteLists;
14+
import org.prism.Nodes.*;
15+
import org.prism.ParsingOptions;
16+
import org.prism.Prism;
17+
18+
import java.util.ArrayList;
19+
import java.util.Arrays;
20+
import java.util.List;
21+
22+
import static org.jruby.api.Convert.asSymbol;
23+
import static org.jruby.lexer.LexingCommon.DOLLAR_UNDERSCORE;
24+
import static org.jruby.parser.ParserType.EVAL;
25+
26+
public class ParserPrismWasm extends ParserPrismBase {
27+
public ParserPrismWasm(Ruby runtime) {
28+
super(runtime);
29+
}
30+
31+
protected byte[] parse(byte[] source, int sourceLength, byte[] metadata) {
32+
try (Prism prism = new Prism()) {
33+
return prism.serialize(metadata, source, sourceLength);
34+
}
35+
}
36+
}

0 commit comments

Comments
 (0)