decoder: Occurrence aware decoder generator

vadl-cli/main/vadl/cli/BaseCommand.java

@@ -450,6 +450,34 @@ private DecoderOptions getDecoderOptions() {
       result.setOptsToSkip(skipOpts.toArray(new DecoderOptions.OptionToSkip[0]));
     }
 
+    var statisticOpts = decoderOptions.stream()
+        .filter(DecoderStatistics.class::isInstance)
+        .map(DecoderStatistics.class::cast)
+        .map(DecoderStatistics::stats)
+        .toList();
+
+    if (statisticOpts.size() > 1) {
+      throw new IllegalArgumentException("Multiple statistics configuration are not allowed.");
+    }
+
+    if (statisticOpts.size() == 1) {
+      result.setStatistics(statisticOpts.getFirst().getAbsolutePath());
+    }
+
+    var penaltyOpts = decoderOptions.stream()
+        .filter(DecoderPenaltyFactor.class::isInstance)
+        .map(DecoderPenaltyFactor.class::cast)
+        .map(DecoderPenaltyFactor::penalty)
+        .toList();
+
+    if (penaltyOpts.size() > 1) {
+      throw new IllegalArgumentException("Multiple penalty configuration are not allowed.");
+    }
+
+    if (penaltyOpts.size() == 1) {
+      result.setMemoryPenalty(penaltyOpts.getFirst());
+    }
+
     return result;
   }
 }

vadl-cli/main/vadl/cli/Helpers.java

@@ -16,6 +16,7 @@
 
 package vadl.cli;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
@@ -24,6 +25,7 @@
 import java.util.Stack;
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.stream.Streams;
 import picocli.CommandLine;
 import vadl.OpenVadlProperties;
@@ -117,11 +119,20 @@ record DecoderStrategy(DecoderOptions.Generator generator) implements DecoderOpt
 record DecoderSkipOption(DecoderOptions.OptionToSkip option) implements DecoderOpt {
 }
 
+record DecoderPenaltyFactor(Double penalty) implements DecoderOpt {
+}
+
+record DecoderStatistics(File stats) implements DecoderOpt {
+}
+
 class DecoderOptsConverter implements Iterable<String>, CommandLine.ITypeConverter<DecoderOpt> {
 
   static final String KEY_STRATEGY = "strategy";
   static final String KEY_SKIP = "skip";
 
+  static final String KEY_STATS = "statistics";
+  static final String KEY_PENALTY_FACTOR = "penalty";
+
   @Override
   public DecoderOpt convert(String value) throws Exception {
     final String[] fragments = value.split("=", -1);
@@ -156,6 +167,26 @@ public DecoderOpt convert(String value) throws Exception {
                   .map(DecoderOptions.OptionToSkip::getSelector).toList()));
     }
 
+    if (KEY_STATS.equals(fragments[0].trim())) {
+      var statFile = new File(fragments[1].trim());
+      if (!statFile.exists()) {
+        throw new CommandLine.TypeConversionException(
+            "Unable to parse decoder option '%s'. Stats file does not exist".formatted(
+                statFile.getAbsolutePath())
+        );
+      }
+      return new DecoderStatistics(statFile);
+    }
+
+    if (KEY_PENALTY_FACTOR.equals(fragments[0].trim())) {
+      var val = fragments[1].trim();
+      if (!StringUtils.isNumeric(val)) {
+        throw new CommandLine.TypeConversionException(
+            "Unable to parse decoder option '%s'. Penalty factor is not numeric".formatted(val));
+      }
+      return new DecoderPenaltyFactor(Double.parseDouble(val));
+    }
+
     throw new CommandLine.TypeConversionException(
         "Illegal decoder option '%s'. Available options are: %s".formatted(value,
             List.of(KEY_SKIP, KEY_STRATEGY)));
@@ -172,6 +203,10 @@ public static List<String> getOptions() {
       options.add(
           "%n%s=%s (%s)".formatted(KEY_SKIP, skipOption.getSelector(), skipOption.getDesc()));
     }
+    options.add("%n%s=%f (Penalty factor for occurrence aware decoder generator)".formatted(
+        KEY_PENALTY_FACTOR, 1.0));
+    options.add(
+        "%n%s=%s (Instruction occurrence statistics)".formatted(KEY_STATS, "/insn-stats.json"));
     return options;
   }
 

vadl/main/vadl/configuration/DecoderOptions.java

@@ -28,9 +28,14 @@ public class DecoderOptions {
    * The possible VADL decode tree generator strategies to choose from.
    */
   public enum Generator {
+    /**
+     * For now only supported by the RTL backend target, since HW is the only target which would
+     * now require sequential lookup. Supports only regular encodings.
+     */
+    TABLE("Lookup table based decoder", "table"),
     REGULAR("Regular decoder generator", "regular"),
     IRREGULAR("Irregular decoder generator, default", "irregular"),
-    RTL_TABLE("RTL table based decoder", "rtl-table"),;
+    OCCURRENCE_AWARE("Occurrence aware generator", "occ-aware");
 
     private final String selector;
     private final String desc;
@@ -77,7 +82,10 @@ public enum OptionToSkip {
         "constraint-synthesis"),
 
     OPT_ENCODING_VERIFICATION("Skip the encoding verification, default: enabled",
-        "encoding-verification");
+        "encoding-verification"),
+
+    OPT_DECODER_VERIFICATION("Skip the correctness verification of the decode tree, default: "
+                                 + "enabled", "decoder-verification");
 
     private final String selector;
     private final String desc;
@@ -118,9 +126,18 @@ public static OptionToSkip fromSelector(String selector) {
   private OptionToSkip[] optsToSkip;
   private Generator generator;
 
+  /**
+   * Options for occurrence aware decoder generator.
+   */
+  private double memoryPenalty;
+
+  @Nullable
+  private String statistics;
+
   public DecoderOptions() {
     optsToSkip = new OptionToSkip[0];
     generator = Generator.IRREGULAR;
+    memoryPenalty = 1.0;
   }
 
   public OptionToSkip[] getOptsToSkip() {
@@ -139,6 +156,43 @@ public void setGenerator(Generator generator) {
     this.generator = generator;
   }
 
+  public Double getMemoryPenalty() {
+    return memoryPenalty;
+  }
+
+  public void setMemoryPenalty(Double memoryPenalty) {
+    this.memoryPenalty = memoryPenalty;
+  }
+
+  @Nullable
+  public String getStatistics() {
+    return statistics;
+  }
+
+  public void setStatistics(@Nullable String statistics) {
+    this.statistics = statistics;
+  }
+
+  public DecoderOptions withGenerator(Generator generator) {
+    setGenerator(generator);
+    return this;
+  }
+
+  public DecoderOptions withOptsToSkip(OptionToSkip... optsToSkip) {
+    setOptsToSkip(optsToSkip);
+    return this;
+  }
+
+  public DecoderOptions withMemoryPenalty(Double memoryPenalty) {
+    setMemoryPenalty(memoryPenalty);
+    return this;
+  }
+
+  public DecoderOptions withStatistics(@Nullable String statistics) {
+    setStatistics(statistics);
+    return this;
+  }
+
   @Override
   public String toString() {
     return "DecoderOptions{"

vadl/main/vadl/dump/infoEnrichers/VdtEnricherCollection.java

@@ -22,6 +22,10 @@
 import vadl.dump.InfoEnricher;
 import vadl.dump.InfoUtils;
 import vadl.dump.entities.VdtEntity;
+import vadl.pass.PassResults;
+import vadl.vdt.impl.irregular.model.DecodeEntry;
+import vadl.vdt.passes.VdtConstraintSynthesisPass;
+import vadl.vdt.passes.VdtInputPreparationPass;
 import vadl.vdt.target.common.DecisionTreeStatsCalculator;
 import vadl.vdt.target.dump.DotGraphGenerator;
 import vadl.vdt.target.dump.InsnDecisionTableGenerator;
@@ -108,21 +112,36 @@ public class VdtEnricherCollection {
         entity.addInfo(info);
       });
 
+  @SuppressWarnings("unchecked")
   public static InfoEnricher VDT_STATS_EXPANDABLE =
       InfoEnricher.forType(VdtEntity.class, (entity, passResults) -> {
 
         var stats = DecisionTreeStatsCalculator.statistics(entity.tree());
 
         final var statsTable = new ArrayList<List<String>>();
 
-        statsTable.add(List.of("Property", "Number of Nodes", "Number of Leaves (Instructions)",
-            "Minimum Depth",
-            "Maximal Depth", "Average Depth", "Longest instruction width"));
-        statsTable.add(List.of("Value", String.valueOf(stats.getNumberOfNodes()),
-            String.valueOf(stats.getNumberOfLeafNodes()), String.valueOf(stats.getMinDepth()),
-            String.valueOf(stats.getMaxDepth()),
-            String.valueOf(Math.round(stats.getAvgDepth() * 100) / 100.0),
-            stats.getMaxInstructionWidth() + " bit"));
+        final List<String> categories = new ArrayList<>(
+            List.of("Property", "Number of Nodes", "Number of Instructions",
+                "Number of Leaves", "Minimum Depth", "Maximal Depth", "Average Depth"));
+        if (stats.getOccurrenceProbability() > 0.0) {
+          categories.add("Weighted Average Depth");
+        }
+        categories.add("Longest instruction width");
+        statsTable.add(categories);
+
+        final List<String> values = new ArrayList<>(
+            List.of("Value", String.valueOf(stats.getNumberOfNodes()),
+                String.valueOf(getInsnCount(passResults)),
+                String.valueOf(stats.getNumberOfLeafNodes()), String.valueOf(stats.getMinDepth()),
+                String.valueOf(stats.getMaxDepth()),
+                String.valueOf(Math.round(stats.getAvgDepth() * 100) / 100.0))
+        );
+        if (stats.getOccurrenceProbability() > 0.0) {
+          values.add(String.valueOf(Math.round(stats.getWeightedAvgDepth() * 100) / 100.0));
+        }
+        values.add(stats.getMaxInstructionWidth() + " bit");
+
+        statsTable.add(values);
 
         var info = InfoUtils.createTableExpandable("Statistics", statsTable);
         entity.addInfo(info);
@@ -133,13 +152,33 @@ public class VdtEnricherCollection {
 
         var stats = DecisionTreeStatsCalculator.statistics(entity.tree());
 
-        entity.addInfo(Info.Tag.of("Instructions", String.valueOf(stats.getNumberOfLeafNodes())));
+        entity.addInfo(Info.Tag.of("Instructions", String.valueOf(getInsnCount(passResults))));
         entity.addInfo(Info.Tag.of("Nodes", String.valueOf(stats.getNumberOfNodes())));
+        entity.addInfo(Info.Tag.of("Leaves", String.valueOf(stats.getNumberOfLeafNodes())));
         entity.addInfo(Info.Tag.of("Max Depth", String.valueOf(stats.getMaxDepth())));
         entity.addInfo(Info.Tag.of("Avg Depth",
             String.valueOf(Math.round(stats.getAvgDepth() * 100) / 100.0)));
+
+        if (stats.getOccurrenceProbability() > 0.0) {
+          entity.addInfo(Info.Tag.of("Weighted Avg Depth",
+              String.valueOf(Math.round(stats.getWeightedAvgDepth() * 100) / 100.0)));
+        }
       });
 
+  @SuppressWarnings("unchecked")
+  private static int getInsnCount(PassResults passResults) {
+    final List<DecodeEntry> entries;
+    if (passResults.hasRunPassOnce(VdtConstraintSynthesisPass.class)) {
+      entries =
+          (List<DecodeEntry>) passResults.lastNullableResultOf(
+              VdtConstraintSynthesisPass.class);
+    } else {
+      entries =
+          (List<DecodeEntry>) passResults.lastNullableResultOf(VdtInputPreparationPass.class);
+    }
+    return entries != null ? entries.size() : 0;
+  }
+
   public static List<InfoEnricher> all = List.of(
       VDT_STATS_TAGS,
       VDT_STATS_EXPANDABLE,

vadl/main/vadl/pass/PassOrders.java

@@ -16,15 +16,17 @@
 
 package vadl.pass;
 
-import static vadl.configuration.DecoderOptions.Generator.RTL_TABLE;
+import static vadl.configuration.DecoderOptions.Generator.TABLE;
+import static vadl.configuration.DecoderOptions.OptionToSkip.OPT_CONSTRAINT_SYNTHESIS;
+import static vadl.configuration.DecoderOptions.OptionToSkip.OPT_DECODER_VERIFICATION;
+import static vadl.configuration.DecoderOptions.OptionToSkip.OPT_ENCODING_VERIFICATION;
 import static vadl.iss.template.IssDefaultRenderingPass.issDefault;
 
 import com.google.common.collect.Streams;
 import java.io.IOException;
 import java.util.Optional;
 import java.util.stream.Stream;
 import javax.annotation.Nullable;
-import vadl.configuration.DecoderOptions;
 import vadl.configuration.GcbConfiguration;
 import vadl.configuration.GeneralConfiguration;
 import vadl.configuration.IssConfiguration;
@@ -144,6 +146,7 @@
 import vadl.vdt.passes.VdtEncodingSemanticVerificationPass;
 import vadl.vdt.passes.VdtInputPreparationPass;
 import vadl.vdt.passes.VdtLoweringPass;
+import vadl.vdt.passes.VdtVerificationPass;
 import vadl.viam.Specification;
 import vadl.viam.passes.ControlFlowOptimizationPass;
 import vadl.viam.passes.DetectRegisterIndicesPass;
@@ -187,6 +190,10 @@ public static PassOrder check(GeneralConfiguration configuration) {
     // check if VDT can be constructed
     addDecodePasses(order, configuration);
 
+    addHtmlDump(order, configuration,
+        "VDT Creation",
+        "Dump directly after VDT generation.");
+
     return order;
   }
 
@@ -628,18 +635,24 @@ private static void addDecodePasses(PassOrder order, GeneralConfiguration config
         .add(new VdtInputPreparationPass(config));
 
     var skipSynthesis = Stream.of(config.getDecoderOptions().getOptsToSkip())
-        .anyMatch(o -> o == DecoderOptions.OptionToSkip.OPT_CONSTRAINT_SYNTHESIS);
+        .anyMatch(o -> o == OPT_CONSTRAINT_SYNTHESIS);
     if (!skipSynthesis) {
       order.add(new VdtConstraintSynthesisPass(config));
     }
 
-    var skipVerification = Stream.of(config.getDecoderOptions().getOptsToSkip())
-        .anyMatch(o -> o == DecoderOptions.OptionToSkip.OPT_ENCODING_VERIFICATION);
-    if (!skipVerification) {
+    var skipEncodingVerification = Stream.of(config.getDecoderOptions().getOptsToSkip())
+        .anyMatch(o -> o == OPT_ENCODING_VERIFICATION);
+    if (!skipEncodingVerification) {
       order.add(new VdtEncodingSemanticVerificationPass(config));
     }
 
     order.add(new VdtLoweringPass(config));
+
+    var skipDecoderVerification = Stream.of(config.getDecoderOptions().getOptsToSkip())
+        .anyMatch(o -> o == OPT_DECODER_VERIFICATION);
+    if (!skipDecoderVerification) {
+      order.add(new VdtVerificationPass(config));
+    }
   }
 
   /**
@@ -686,8 +699,8 @@ public static PassOrder rtl(RtlConfiguration config) throws IOException {
         "mia-inline",
         "MiA after inlining instruction behavior");
 
-    if (config.getDecoderOptions().getGenerator() != RTL_TABLE) {
-      // Prepares and constructs the VDT, which is not used by the rtl-table strategy
+    if (config.getDecoderOptions().getGenerator() != TABLE) {
+      // Prepares and constructs the VDT, which is not used by the 'table' strategy
       addDecodePasses(order, config);
     }