Implement Chicory WASM backend for the Ruby API

.github/workflows/java-wasm-bindings.yml

@@ -27,7 +27,7 @@ jobs:
           bundler-cache: true
 
       - name: rake templates
-        run: PRISM_EXCLUDE_PRETTYPRINT=1 PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS=1 bundle exec rake templates
+        run: PRISM_EXCLUDE_PRETTYPRINT=1 PRISM_OMIT_NODE_ID=1 bundle exec rake templates
 
       - name: Set up WASI-SDK
         run: |

Makefile

@@ -31,7 +31,7 @@ all: shared static
 shared: build/libprism.$(SOEXT)
 static: build/libprism.a
 wasm: javascript/src/prism.wasm
-java-wasm: java/wasm/src/main/wasm/prism.wasm
+java-wasm: java/wasm/src/main/wasm/prism.wasm java/wasm-full/src/main/wasm/prism-full.wasm
 
 build/libprism.$(SOEXT): $(SHARED_OBJECTS)
 	$(ECHO) "linking $@ with $(CC)"
@@ -52,6 +52,17 @@ javascript/src/prism.wasm: Makefile $(SOURCES) $(HEADERS)
 		-o $@ $(SOURCES)
 
 java/wasm/src/main/wasm/prism.wasm: Makefile $(SOURCES) $(HEADERS)
+	$(ECHO) "building $@"
+	$(Q) $(MAKEDIRS) $(@D)
+	$(Q) $(WASI_SDK_PATH)/bin/clang \
+		$(DEBUG_FLAGS) \
+		-DPRISM_EXCLUDE_PRETTYPRINT -DPRISM_EXPORT_SYMBOLS -D_WASI_EMULATED_MMAN -DPRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS=1 \
+		-DPRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS \
+		-lwasi-emulated-mman $(CPPFLAGS) $(JAVA_WASM_CFLAGS) \
+		-Wl,--export-all -Wl,--no-entry -mexec-model=reactor -lc++ -lc++abi \
+		-o $@ $(SOURCES)
+
+java/wasm-full/src/main/wasm/prism-full.wasm: Makefile $(SOURCES) $(HEADERS)
 	$(ECHO) "building $@"
 	$(Q) $(MAKEDIRS) $(@D)
 	$(Q) $(WASI_SDK_PATH)/bin/clang \

Rakefile

@@ -55,6 +55,7 @@ CLOBBER.concat(Prism::Template::TEMPLATES)
 CLOBBER.concat(["build"])
 CLOBBER << "lib/prism/prism.#{RbConfig::CONFIG["DLEXT"]}"
 CLOBBER << "java/wasm/src/main/resources/prism.wasm"
+CLOBBER << "java/wasm-full/src/main/resources/prism-full.wasm"
 
 Prism::Template::TEMPLATES.each do |filepath|
   desc "Generate #{filepath}"

Steepfile

@@ -15,4 +15,7 @@ target :lib do
 
   # Ignored because we do not want to overlap with the C extension.
   ignore "lib/prism/ffi.rb"
+  ignore "lib/prism/ffi/common.rb"
+  ignore "lib/prism/ffi/native_ffi.rb"
+  ignore "lib/prism/ffi/wasm_ffi.rb"
 end

java/README.md

@@ -15,7 +15,7 @@ Some files need to be generated before the Maven artifacts can build:
 Sources under `api` are generated from templates in `../templates`. Those sources are generated using the follow command line:
 
 ```
-$ PRISM_EXCLUDE_PRETTYPRINT=1 PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS=1 bundle exec rake templates
+$ PRISM_EXCLUDE_PRETTYPRINT=1 bundle exec rake templates
 ```
 
 The files are generated under `api/src/main/java-templates` and will not be removed with `mvn clean`.

java/api/pom.xml

@@ -9,7 +9,7 @@
   </parent>
 
   <artifactId>prism-parser-api</artifactId>
-  <name>Java Prism</name>
+  <name>Java Prism API</name>
   <description>Java API for the Prism Ruby language parser</description>
   <url>https://github.com/ruby/prism</url>
 

java/pom.xml

@@ -6,7 +6,7 @@
   <artifactId>prism-parser</artifactId>
   <version>0.0.2-SNAPSHOT</version>
   <packaging>pom</packaging>
-  <name>Java Prism</name>
+  <name>Java Prism parent</name>
   <description>Java API for the Prism Ruby language parser</description>
   <url>https://github.com/ruby/prism</url>
 
@@ -51,6 +51,7 @@
   <modules>
     <module>api</module>
     <module>wasm</module>
+    <module>wasm-full</module>
   </modules>
 
   <profiles>

java/wasm-full/pom.xml

@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.ruby-lang</groupId>
+    <artifactId>prism-parser</artifactId>
+    <version>0.0.2-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>prism-parser-wasm-full</artifactId>
+  <name>Java Prism WASM with full parsed content</name>
+  <description>Java WASM bindings for the Prism parser shared library</description>
+  <url>https://github.com/ruby/prism</url>
+
+  <properties>
+    <chicory.version>1.7.5</chicory.version>
+  </properties>
+
+  <dependencyManagement>
+    <dependencies>
+          <dependency>
+              <groupId>com.dylibso.chicory</groupId>
+              <artifactId>bom</artifactId>
+              <version>${chicory.version}</version>
+              <type>pom</type>
+              <scope>import</scope>
+          </dependency>
+      </dependencies>
+  </dependencyManagement>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.dylibso.chicory</groupId>
+      <artifactId>runtime</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.dylibso.chicory</groupId>
+      <artifactId>log</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.dylibso.chicory</groupId>
+      <artifactId>wasi</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.dylibso.chicory</groupId>
+      <artifactId>wasm</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <version>${junit.version}</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>com.dylibso.chicory</groupId>
+        <artifactId>chicory-compiler-maven-plugin</artifactId>
+        <version>${chicory.version}</version>
+        <executions>
+          <execution>
+            <id>prism</id>
+            <goals>
+              <goal>compile</goal>
+            </goals>
+            <configuration>
+              <name>org.ruby_lang.prism.wasm.full.PrismParser</name>
+              <moduleInterface>org.ruby_lang.prism.wasm.full.Prism</moduleInterface>
+              <wasmFile>src/main/wasm/prism-full.wasm</wasmFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

java/wasm-full/src/main/java/org/ruby_lang/prism/wasm/full/Prism.java

@@ -0,0 +1,153 @@
+package org.ruby_lang.prism.wasm.full;
+
+import com.dylibso.chicory.runtime.ByteArrayMemory;
+import com.dylibso.chicory.runtime.ImportValues;
+import com.dylibso.chicory.runtime.Instance;
+import com.dylibso.chicory.wasi.WasiOptions;
+import com.dylibso.chicory.wasi.WasiPreview1;
+
+import java.nio.charset.StandardCharsets;
+
+public class Prism implements AutoCloseable {
+    private final WasiPreview1 wasi;
+    protected final Prism_ModuleExports exports;
+    private final Instance instance;
+
+    public Prism() {
+        this(WasiOptions.builder().build());
+    }
+
+    public Prism(WasiOptions wasiOpts) {
+        wasi = WasiPreview1.builder().withOptions(wasiOpts).build();
+        instance = Instance.builder(PrismParser.load())
+            .withMemoryFactory(ByteArrayMemory::new)
+            .withMachineFactory(PrismParser::create)
+            .withImportValues(ImportValues.builder().addFunction(wasi.toHostFunctions()).build())
+            .build();
+        exports = new Prism_ModuleExports(instance);
+    }
+
+    public String version() {
+        int versionPointer = exports.pmVersion();
+        int length = exports.strchr(versionPointer, 0);
+
+        return new String(instance.memory().readBytes(versionPointer, length - versionPointer));
+    }
+
+    public byte[] parse(byte[] sourceBytes, byte[] packedOptions) {
+        try (
+            Buffer buffer = new Buffer();
+            Source source = new Source(sourceBytes, 0, sourceBytes.length);
+            Options options = new Options(packedOptions)) {
+
+            return parse(buffer, source, options);
+        }
+    }
+
+    public byte[] lex(byte[] sourceBytes, byte[] packedOptions) {
+        try (
+            Buffer buffer = new Buffer();
+            Source source = new Source(sourceBytes, 0, sourceBytes.length);
+            Options options = new Options(packedOptions)) {
+
+            return lex(buffer, source, options);
+        }
+    }
+
+    public byte[] parse(byte[] sourceBytes, int sourceOffset, int sourceLength, byte[] packedOptions) {
+        try (
+            Buffer buffer = new Buffer();
+            Source source = new Source(sourceBytes, sourceOffset, sourceLength);
+            Options options = new Options(packedOptions)) {
+
+            return parse(buffer, source, options);
+        }
+    }
+
+    public byte[] parse(Buffer buffer, Source source, Options options) {
+        exports.pmSerializeParse(
+            buffer.pointer, source.pointer, source.length, options.pointer);
+
+        return buffer.read();
+    }
+
+    public byte[] lex(Buffer buffer, Source source, Options options) {
+        exports.pmSerializeLex(
+            buffer.pointer, source.pointer, source.length, options.pointer);
+
+        return buffer.read();
+    }
+
+    public class Buffer implements AutoCloseable {
+        final int pointer;
+
+        Buffer() {
+            pointer = exports.pmBufferNew();
+            clear();
+        }
+
+        public void clear() {
+            exports.pmBufferClear(pointer);
+        }
+
+        public void close() {
+            exports.pmBufferFree(pointer);
+        }
+
+        public byte[] read() {
+            return instance.memory().readBytes(
+                exports.pmBufferValue(pointer),
+                exports.pmBufferLength(pointer));
+        }
+    }
+
+    public class Source implements AutoCloseable{
+        final int pointer;
+        final int length;
+
+        public Source(int length) {
+            pointer = exports.calloc(1, length);
+            this.length = length;
+        }
+
+        public Source(byte[] bytes, int offset, int length) {
+            this(length + 1);
+            write(bytes, offset, length);
+        }
+
+        public Source(byte[] bytes) {
+            this(bytes, 0, bytes.length);
+        }
+
+        public void write(byte[] bytes, int offset, int length) {
+            assert length + 1 <= this.length;
+            instance.memory().write(pointer, bytes, offset, length);
+            instance.memory().writeByte(pointer + length, (byte) 0);
+        }
+
+        public void close() {
+            exports.free(pointer);
+        }
+    }
+
+    class Options implements AutoCloseable {
+        final int pointer;
+
+        Options(byte[] packedOptions) {
+            int pointer = exports.calloc(1, packedOptions.length);
+            instance.memory().write(pointer, packedOptions);
+            this.pointer = pointer;
+        }
+
+        public void close() {
+            exports.free(pointer);
+        }
+    }
+
+    @Override
+    public void close() {
+        if (wasi != null) {
+            wasi.close();
+        }
+    }
+}

java/wasm/pom.xml

@@ -9,7 +9,7 @@
   </parent>
 
   <artifactId>prism-parser-wasm</artifactId>
-  <name>Java Prism WASM</name>
+  <name>Java Prism WASM with semantic-only content</name>
   <description>Java WASM bindings for the Prism parser shared library</description>
   <url>https://github.com/ruby/prism</url>