feat: add DWARF debugging support and source mapping infrastructure

.gitattributes

@@ -0,0 +1 @@
+runtime/src/test/resources/example.smap eol=lf 

build-time-compiler/src/main/java/com/dylibso/chicory/build/time/compiler/Config.java

@@ -1,6 +1,7 @@
 package com.dylibso.chicory.build.time.compiler;
 
 import com.dylibso.chicory.compiler.InterpreterFallback;
+import com.dylibso.chicory.runtime.DebugParser;
 import java.nio.file.Path;
 import java.util.Set;
 import java.util.StringJoiner;
@@ -41,21 +42,28 @@ public final class Config {
      */
     private final Set<Integer> interpretedFunctions;
 
+    /**
+     * the debug parser to use to extract debug symbols from the wasm module.
+     */
+    private final DebugParser debugParser;
+
     private Config(
             Path wasmFile,
             String name,
             Path targetClassFolder,
             Path targetSourceFolder,
             Path targetWasmFolder,
             InterpreterFallback interpreterFallback,
-            Set<Integer> interpretedFunctions) {
+            Set<Integer> interpretedFunctions,
+            DebugParser debugParser) {
         this.wasmFile = wasmFile;
         this.name = name;
         this.targetClassFolder = targetClassFolder;
         this.targetSourceFolder = targetSourceFolder;
         this.targetWasmFolder = targetWasmFolder;
         this.interpreterFallback = interpreterFallback;
         this.interpretedFunctions = interpretedFunctions;
+        this.debugParser = debugParser;
     }
 
     public Path wasmFile() {
@@ -90,6 +98,10 @@ public static Builder builder() {
         return new Builder();
     }
 
+    public DebugParser debugParser() {
+        return debugParser;
+    }
+
     @SuppressWarnings("StringSplitter")
     public String getPackageName() {
         var split = name.split("\\.");
@@ -114,6 +126,7 @@ public static final class Builder {
         private Path targetWasmFolder;
         private InterpreterFallback interpreterFallback = InterpreterFallback.FAIL;
         private Set<Integer> interpretedFunctions;
+        private DebugParser debugParser;
 
         private Builder() {}
 
@@ -152,6 +165,11 @@ public Builder withInterpretedFunctions(Set<Integer> interpretedFunctions) {
             return this;
         }
 
+        public Builder withDebugParser(DebugParser debugParser) {
+            this.debugParser = debugParser;
+            return this;
+        }
+
         public Config build() {
             return new Config(
                     wasmFile,
@@ -160,7 +178,8 @@ public Config build() {
                     targetSourceFolder,
                     targetWasmFolder,
                     interpreterFallback,
-                    interpretedFunctions);
+                    interpretedFunctions,
+                    debugParser);
         }
     }
 }

build-time-compiler/src/main/java/com/dylibso/chicory/build/time/compiler/Generator.java

@@ -8,12 +8,14 @@
 import com.dylibso.chicory.compiler.internal.Compiler;
 import com.dylibso.chicory.runtime.Instance;
 import com.dylibso.chicory.runtime.Machine;
+import com.dylibso.chicory.runtime.Stratum;
 import com.dylibso.chicory.wasm.Parser;
 import com.dylibso.chicory.wasm.WasmModule;
 import com.dylibso.chicory.wasm.WasmWriter;
 import com.dylibso.chicory.wasm.types.OpCode;
 import com.dylibso.chicory.wasm.types.RawSection;
 import com.dylibso.chicory.wasm.types.SectionId;
+import com.dylibso.chicory.wasm.types.UnknownCustomSection;
 import com.github.javaparser.ast.CompilationUnit;
 import com.github.javaparser.ast.Modifier;
 import com.github.javaparser.ast.NodeList;
@@ -36,6 +38,7 @@
 import java.io.InputStream;
 import java.io.UncheckedIOException;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Map;
@@ -55,6 +58,7 @@ public Set<Integer> generateResources() throws IOException {
         var compiler =
                 Compiler.builder(module)
                         .withClassName(machineName)
+                        .withDebugParser(config.debugParser())
                         .withInterpreterFallback(config.interpreterFallback())
                         .withInterpretedFunctions(config.interpretedFunctions())
                         .build();
@@ -70,6 +74,12 @@ public Set<Integer> generateResources() throws IOException {
             Files.write(targetFile, entry.getValue());
         }
 
+        for (Map.Entry<String, byte[]> entry : result.extraResources().entrySet()) {
+            var resourceName = entry.getKey();
+            var targetFile = config.targetWasmFolder().resolve(resourceName);
+            Files.write(targetFile, entry.getValue());
+        }
+
         return result.interpretedFunctions();
     }
 
@@ -117,15 +127,34 @@ public void generateMetaWasm(Set<Integer> interpretedFunctions) throws IOExcepti
                         var out = new ByteArrayOutputStream();
                         var importFuncs = module.importSection().importCount();
                         int count = module.codeSection().functionBodyCount();
+
                         writeVarUInt32(out, count);
+
+                        var inputStratum = Stratum.builder("WASM").build();
+                        if (config.debugParser() != null) {
+                            inputStratum = config.debugParser().apply(module);
+                        }
+                        final var outputStratum = Stratum.builder("WASM");
+
                         var actual = readVarUInt32(source);
                         assert count == actual;
                         for (int i = 0; i < count; i++) {
                             var funcId = importFuncs + i;
                             if (interpretedFunctions.contains(funcId)) {
+                                var bodySize = (int) readVarUInt32(source);
+
+                                if (!inputStratum.isEmpty()) {
+                                    var newFunctionAddress = out.size();
+                                    remapFunctionStratum(
+                                            module,
+                                            funcId,
+                                            inputStratum,
+                                            outputStratum,
+                                            newFunctionAddress,
+                                            bodySize);
+                                }
 
                                 // Copy over the original function body from the source
-                                var bodySize = (int) readVarUInt32(source);
                                 writeVarUInt32(out, bodySize);
                                 var bodyBytes = new byte[bodySize];
                                 source.get(bodyBytes);
@@ -151,6 +180,14 @@ public void generateMetaWasm(Set<Integer> interpretedFunctions) throws IOExcepti
                             }
                         }
                         writer.writeSection(SectionId.CODE, out.toByteArray());
+                        if (!outputStratum.isEmpty()) {
+                            var smap = Stratum.toSMapString("interpreted.wasm", outputStratum);
+                            writer.writeSection(
+                                    UnknownCustomSection.builder()
+                                            .withName("SMAP")
+                                            .withBytes(smap.getBytes(StandardCharsets.UTF_8))
+                                            .build());
+                        }
                     } else if (section.sectionId() != SectionId.CUSTOM) {
                         writer.writeSection((RawSection) section);
                     }
@@ -164,6 +201,64 @@ public void generateMetaWasm(Set<Integer> interpretedFunctions) throws IOExcepti
         Files.write(newWasmFile, writer.bytes());
     }
 
+    private static void remapFunctionStratum(
+            WasmModule module,
+            int funcId,
+            Stratum inputStratum,
+            Stratum.Builder outputStratum,
+            int newFunctionAddress,
+            int functionSize) {
+        String debugFunctionName = null;
+        Stratum.Line lastLine = null;
+        var lastLineAddress = -1;
+        var originalFunctionAddress = -1;
+
+        var instructions = module.codeSection().getFunctionBody(funcId).instructions();
+        for (var ins : instructions) {
+            var relativeAddress = ins.address() - module.codeSection().address();
+
+            if (originalFunctionAddress == -1) {
+                originalFunctionAddress = relativeAddress;
+            }
+
+            if (debugFunctionName == null) {
+                debugFunctionName = inputStratum.getFunctionMapping(relativeAddress);
+                outputStratum.withFunctionMapping(
+                        debugFunctionName, newFunctionAddress, newFunctionAddress + functionSize);
+            }
+
+            var line = inputStratum.getInputLine(relativeAddress);
+            if (line != null) {
+                // did we get a new line mapping?
+                if (lastLine != null && !line.equals(lastLine)) {
+                    var newAddress = lastLineAddress - originalFunctionAddress + newFunctionAddress;
+                    var instructionSize = relativeAddress - lastLineAddress;
+                    outputStratum.withLineMapping(
+                            lastLine.fileName(),
+                            lastLine.filePath(),
+                            lastLine.line(),
+                            lastLine.count(),
+                            newAddress,
+                            instructionSize);
+                }
+                lastLine = line;
+                lastLineAddress = relativeAddress;
+            }
+        }
+
+        if (lastLine != null) {
+            var newAddress = lastLineAddress - originalFunctionAddress + newFunctionAddress;
+            var instructionSize = 1;
+            outputStratum.withLineMapping(
+                    lastLine.fileName(),
+                    lastLine.filePath(),
+                    lastLine.line(),
+                    lastLine.count(),
+                    newAddress,
+                    instructionSize);
+        }
+    }
+
     private static void createFolders(Path filesFolder, String[] split) throws IOException {
         for (int i = 0; i < (split.length - 1); i++) {
             filesFolder = filesFolder.resolve(split[i]);

compiler-maven-plugin/pom.xml

@@ -22,6 +22,10 @@
       <groupId>com.dylibso.chicory</groupId>
       <artifactId>compiler</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.dylibso.chicory</groupId>
+      <artifactId>runtime</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.maven</groupId>
       <artifactId>maven-core</artifactId>

compiler-maven-plugin/src/main/java/com/dylibso/chicory/build/time/maven/ChicoryCompilerGenMojo.java

@@ -3,8 +3,11 @@
 import com.dylibso.chicory.build.time.compiler.Config;
 import com.dylibso.chicory.build.time.compiler.Generator;
 import com.dylibso.chicory.compiler.InterpreterFallback;
+import com.dylibso.chicory.runtime.DebugParser;
 import java.io.File;
 import java.io.IOException;
+import java.util.ServiceConfigurationError;
+import java.util.ServiceLoader;
 import java.util.Set;
 import org.apache.maven.model.Resource;
 import org.apache.maven.plugin.AbstractMojo;
@@ -76,6 +79,19 @@ public class ChicoryCompilerGenMojo extends AbstractMojo {
 
     @Override
     public void execute() throws MojoExecutionException {
+
+        DebugParser debugParser = null;
+        // Use the DebugParser if it's on the classpath.
+        try {
+            for (var service : ServiceLoader.load(DebugParser.class)) {
+                debugParser = service;
+                getLog().info("Using debug parser: " + debugParser.getClass().getName());
+                break;
+            }
+        } catch (ServiceConfigurationError ignore) {
+            debugParser = null;
+        }
+
         getLog().info("Generating AOT classes for " + name + " from " + wasmFile);
 
         var config =
@@ -87,6 +103,7 @@ public void execute() throws MojoExecutionException {
                         .withTargetWasmFolder(targetWasmFolder.toPath())
                         .withInterpreterFallback(interpreterFallback)
                         .withInterpretedFunctions(interpretedFunctions)
+                        .withDebugParser(debugParser)
                         .build();
 
         var generator = new Generator(config);

compiler-tests/src/test/java/com/dylibso/chicory/testing/MethodTooLargeTest.java

@@ -20,14 +20,15 @@
 import org.objectweb.asm.util.TraceClassVisitor;
 
 public class MethodTooLargeTest {
+    public static final String CLASS_NAME = "com.dylibso.chicory.$gen.CompiledMachine";
 
     @Test
     public void testBigFunc() {
         String wat = methodTooLarge(20_000);
         byte[] wasm = Wat2Wasm.parse(wat);
 
         var module = Parser.parse(wasm);
-        var result = Compiler.builder(module).build().compile();
+        var result = Compiler.builder(module).withClassName(CLASS_NAME).build().compile();
 
         // We only verify that the resulting class contains the fallback to interpreter
         verifyClass(result.classBytes(), true);

compiler-tests/src/test/resources/com/dylibso/chicory/testing/MethodTooLargeTest.testBigFunc.approved.txt

@@ -3,7 +3,14 @@ private final Lcom/dylibso/chicory/runtime/Instance; instance
   private final Lcom/dylibso/chicory/runtime/internal/CompilerInterpreterMachine; compilerInterpreterMachine
 }
 
+  public final static INNERCLASS com/dylibso/chicory/runtime/Stratum$Line com/dylibso/chicory/runtime/Stratum Line
+  public final static INNERCLASS java/lang/invoke/MethodHandles$Lookup java/lang/invoke/MethodHandles Lookup
+
   private final static Z memCopyWorkaround
+
+  private static [Lcom/dylibso/chicory/runtime/Stratum; STRATA_BY_FUNC_GROUP
+
+  private static Ljava/lang/String; funcGroupClassPrefix
 }
 
 final class com/dylibso/chicory/$gen/CompiledMachineFuncGroup_0 {

compiler/src/main/java/com/dylibso/chicory/compiler/MachineFactoryCompiler.java

@@ -1,6 +1,7 @@
 package com.dylibso.chicory.compiler;
 
 import com.dylibso.chicory.compiler.internal.MachineFactory;
+import com.dylibso.chicory.runtime.DebugParser;
 import com.dylibso.chicory.runtime.Instance;
 import com.dylibso.chicory.runtime.Machine;
 import com.dylibso.chicory.wasm.WasmModule;
@@ -25,19 +26,20 @@ private MachineFactoryCompiler() {
      * </pre>
      * <p>
      * Every instance created by the builder will pay the cost of compiling the module.
-     * </p>
      *
      * @see #compile(WasmModule) If you want to compile the module only once for multiple instances.
      */
     public static Machine compile(Instance instance) {
-        return compile(instance.module()).apply(instance);
+        return builder(instance.module())
+                .withDebugParser(instance.debugParser())
+                .compile()
+                .apply(instance);
     }
 
     /**
      * Compiles a machine factory that can used in instance builders.
      * The module is only compiled once and the machine factory is reused for every
      * instance created by the builder.
-     * <p>
      * <pre>
      * var module  = Parser.parse(is);
      * var builder = Instance.builder(module)
@@ -55,7 +57,6 @@ public static Function<Instance, Machine> compile(WasmModule module) {
      * The builder allows you to configure the compiler options used to compile the module to
      * byte code.
      * This should be used when you want to create multiple instances of the same module.
-     * <p>
      * <pre>
      * var module  = Parser.parse(is);
      * var builder = Instance.builder(module)
@@ -101,6 +102,11 @@ public Builder withInterpretedFunctions(Set<Integer> interpretedFunctions) {
             return this;
         }
 
+        public Builder withDebugParser(DebugParser debugParser) {
+            compilerBuilder.withDebugParser(debugParser);
+            return this;
+        }
+
         public Function<Instance, Machine> compile() {
             var result = compilerBuilder.build().compile();
             return new MachineFactory(module, result.machineFactory());