decoder: extend CLI with new decoder options

- extend options with memory penalty and
  instruction statistics path
- generator fixes: pattern splitting and
  decision function generation

Author: rascmatt

vadl-cli/main/vadl/cli/BaseCommand.java

@@ -450,6 +450,34 @@ private DecoderOptions getDecoderOptions() {
       result.setOptsToSkip(skipOpts.toArray(new DecoderOptions.OptionToSkip[0]));
     }
 
+    var statisticOpts = decoderOptions.stream()
+        .filter(DecoderStatistics.class::isInstance)
+        .map(DecoderStatistics.class::cast)
+        .map(DecoderStatistics::stats)
+        .toList();
+
+    if (statisticOpts.size() > 1) {
+      throw new IllegalArgumentException("Multiple statistics configuration are not allowed.");
+    }
+
+    if (statisticOpts.size() == 1) {
+      result.setStatistics(statisticOpts.getFirst().getAbsolutePath());
+    }
+
+    var penaltyOpts = decoderOptions.stream()
+        .filter(DecoderPenaltyFactor.class::isInstance)
+        .map(DecoderPenaltyFactor.class::cast)
+        .map(DecoderPenaltyFactor::penalty)
+        .toList();
+
+    if (penaltyOpts.size() > 1) {
+      throw new IllegalArgumentException("Multiple penalty configuration are not allowed.");
+    }
+
+    if (penaltyOpts.size() == 1) {
+      result.setMemoryPenalty(penaltyOpts.getFirst());
+    }
+
     return result;
   }
 }

vadl-cli/main/vadl/cli/Helpers.java

@@ -16,6 +16,7 @@
 
 package vadl.cli;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
@@ -24,6 +25,7 @@
 import java.util.Stack;
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.stream.Streams;
 import picocli.CommandLine;
 import vadl.OpenVadlProperties;
@@ -117,11 +119,20 @@ record DecoderStrategy(DecoderOptions.Generator generator) implements DecoderOpt
 record DecoderSkipOption(DecoderOptions.OptionToSkip option) implements DecoderOpt {
 }
 
+record DecoderPenaltyFactor(Double penalty) implements DecoderOpt {
+}
+
+record DecoderStatistics(File stats) implements DecoderOpt {
+}
+
 class DecoderOptsConverter implements Iterable<String>, CommandLine.ITypeConverter<DecoderOpt> {
 
   static final String KEY_STRATEGY = "strategy";
   static final String KEY_SKIP = "skip";
 
+  static final String KEY_STATS = "statistics";
+  static final String KEY_PENALTY_FACTOR = "penalty";
+
   @Override
   public DecoderOpt convert(String value) throws Exception {
     final String[] fragments = value.split("=", -1);
@@ -156,6 +167,26 @@ public DecoderOpt convert(String value) throws Exception {
                   .map(DecoderOptions.OptionToSkip::getSelector).toList()));
     }
 
+    if (KEY_STATS.equals(fragments[0].trim())) {
+      var statFile = new File(fragments[1].trim());
+      if (!statFile.exists()) {
+        throw new CommandLine.TypeConversionException(
+            "Unable to parse decoder option '%s'. Stats file does not exist".formatted(
+                statFile.getAbsolutePath())
+        );
+      }
+      return new DecoderStatistics(statFile);
+    }
+
+    if (KEY_PENALTY_FACTOR.equals(fragments[0].trim())) {
+      var val = fragments[1].trim();
+      if (!StringUtils.isNumeric(val)) {
+        throw new CommandLine.TypeConversionException(
+            "Unable to parse decoder option '%s'. Penalty factor is not numeric".formatted(val));
+      }
+      return new DecoderPenaltyFactor(Double.parseDouble(val));
+    }
+
     throw new CommandLine.TypeConversionException(
         "Illegal decoder option '%s'. Available options are: %s".formatted(value,
             List.of(KEY_SKIP, KEY_STRATEGY)));
@@ -172,6 +203,10 @@ public static List<String> getOptions() {
       options.add(
           "%n%s=%s (%s)".formatted(KEY_SKIP, skipOption.getSelector(), skipOption.getDesc()));
     }
+    options.add("%n%s=%f (Penalty factor for occurrence aware decoder generator)".formatted(
+        KEY_PENALTY_FACTOR, 1.0));
+    options.add(
+        "%n%s=%s (Instruction occurrence statistics)".formatted(KEY_STATS, "/insn-stats.json"));
     return options;
   }
 

vadl/main/vadl/configuration/DecoderOptions.java

@@ -28,10 +28,14 @@ public class DecoderOptions {
    * The possible VADL decode tree generator strategies to choose from.
    */
   public enum Generator {
+    /**
+     * For now only supported by the RTL backend target, since HW is the only target which would
+     * now require sequential lookup. Supports only regular encodings.
+     */
+    TABLE("Lookup table based decoder", "table"),
     REGULAR("Regular decoder generator", "regular"),
     IRREGULAR("Irregular decoder generator, default", "irregular"),
-    RTL_TABLE("RTL table based decoder", "rtl-table"),
-    OCC("Irregular decoder generator, using occurrence probabilities", "occ");
+    OCCURRENCE_AWARE("Occurrence aware generator", "occ-aware");
 
     private final String selector;
     private final String desc;
@@ -122,9 +126,18 @@ public static OptionToSkip fromSelector(String selector) {
   private OptionToSkip[] optsToSkip;
   private Generator generator;
 
+  /**
+   * Options for occurrence aware decoder generator.
+   */
+  private double memoryPenalty;
+
+  @Nullable
+  private String statistics;
+
   public DecoderOptions() {
     optsToSkip = new OptionToSkip[0];
     generator = Generator.IRREGULAR;
+    memoryPenalty = 1.0;
   }
 
   public OptionToSkip[] getOptsToSkip() {
@@ -143,6 +156,23 @@ public void setGenerator(Generator generator) {
     this.generator = generator;
   }
 
+  public Double getMemoryPenalty() {
+    return memoryPenalty;
+  }
+
+  public void setMemoryPenalty(Double memoryPenalty) {
+    this.memoryPenalty = memoryPenalty;
+  }
+
+  @Nullable
+  public String getStatistics() {
+    return statistics;
+  }
+
+  public void setStatistics(@Nullable String statistics) {
+    this.statistics = statistics;
+  }
+
   public DecoderOptions withGenerator(Generator generator) {
     setGenerator(generator);
     return this;
@@ -153,6 +183,16 @@ public DecoderOptions withOptsToSkip(OptionToSkip... optsToSkip) {
     return this;
   }
 
+  public DecoderOptions withMemoryPenalty(Double memoryPenalty) {
+    setMemoryPenalty(memoryPenalty);
+    return this;
+  }
+
+  public DecoderOptions withStatistics(@Nullable String statistics) {
+    setStatistics(statistics);
+    return this;
+  }
+
   @Override
   public String toString() {
     return "DecoderOptions{"

vadl/main/vadl/dump/infoEnrichers/VdtEnricherCollection.java

@@ -22,6 +22,10 @@
 import vadl.dump.InfoEnricher;
 import vadl.dump.InfoUtils;
 import vadl.dump.entities.VdtEntity;
+import vadl.pass.PassResults;
+import vadl.vdt.impl.irregular.model.DecodeEntry;
+import vadl.vdt.passes.VdtConstraintSynthesisPass;
+import vadl.vdt.passes.VdtInputPreparationPass;
 import vadl.vdt.target.common.DecisionTreeStatsCalculator;
 import vadl.vdt.target.dump.DotGraphGenerator;
 import vadl.vdt.target.dump.InsnDecisionTableGenerator;
@@ -108,21 +112,36 @@ public class VdtEnricherCollection {
         entity.addInfo(info);
       });
 
+  @SuppressWarnings("unchecked")
   public static InfoEnricher VDT_STATS_EXPANDABLE =
       InfoEnricher.forType(VdtEntity.class, (entity, passResults) -> {
 
         var stats = DecisionTreeStatsCalculator.statistics(entity.tree());
 
         final var statsTable = new ArrayList<List<String>>();
 
-        statsTable.add(List.of("Property", "Number of Nodes", "Number of Leaves (Instructions)",
-            "Minimum Depth",
-            "Maximal Depth", "Average Depth", "Longest instruction width"));
-        statsTable.add(List.of("Value", String.valueOf(stats.getNumberOfNodes()),
-            String.valueOf(stats.getNumberOfLeafNodes()), String.valueOf(stats.getMinDepth()),
-            String.valueOf(stats.getMaxDepth()),
-            String.valueOf(Math.round(stats.getAvgDepth() * 100) / 100.0),
-            stats.getMaxInstructionWidth() + " bit"));
+        final List<String> categories = new ArrayList<>(
+            List.of("Property", "Number of Nodes", "Number of Instructions",
+                "Number of Leaves", "Minimum Depth", "Maximal Depth", "Average Depth"));
+        if (stats.getOccurrenceProbability() > 0.0) {
+          categories.add("Weighted Average Depth");
+        }
+        categories.add("Longest instruction width");
+        statsTable.add(categories);
+
+        final List<String> values = new ArrayList<>(
+            List.of("Value", String.valueOf(stats.getNumberOfNodes()),
+                String.valueOf(getInsnCount(passResults)),
+                String.valueOf(stats.getNumberOfLeafNodes()), String.valueOf(stats.getMinDepth()),
+                String.valueOf(stats.getMaxDepth()),
+                String.valueOf(Math.round(stats.getAvgDepth() * 100) / 100.0))
+        );
+        if (stats.getOccurrenceProbability() > 0.0) {
+          values.add(String.valueOf(Math.round(stats.getWeightedAvgDepth() * 100) / 100.0));
+        }
+        values.add(stats.getMaxInstructionWidth() + " bit");
+
+        statsTable.add(values);
 
         var info = InfoUtils.createTableExpandable("Statistics", statsTable);
         entity.addInfo(info);
@@ -133,13 +152,33 @@ public class VdtEnricherCollection {
 
         var stats = DecisionTreeStatsCalculator.statistics(entity.tree());
 
-        entity.addInfo(Info.Tag.of("Instructions", String.valueOf(stats.getNumberOfLeafNodes())));
+        entity.addInfo(Info.Tag.of("Instructions", String.valueOf(getInsnCount(passResults))));
         entity.addInfo(Info.Tag.of("Nodes", String.valueOf(stats.getNumberOfNodes())));
+        entity.addInfo(Info.Tag.of("Leaves", String.valueOf(stats.getNumberOfLeafNodes())));
         entity.addInfo(Info.Tag.of("Max Depth", String.valueOf(stats.getMaxDepth())));
         entity.addInfo(Info.Tag.of("Avg Depth",
             String.valueOf(Math.round(stats.getAvgDepth() * 100) / 100.0)));
+
+        if (stats.getOccurrenceProbability() > 0.0) {
+          entity.addInfo(Info.Tag.of("Weighted Avg Depth",
+              String.valueOf(Math.round(stats.getWeightedAvgDepth() * 100) / 100.0)));
+        }
       });
 
+  @SuppressWarnings("unchecked")
+  private static int getInsnCount(PassResults passResults) {
+    final List<DecodeEntry> entries;
+    if (passResults.hasRunPassOnce(VdtConstraintSynthesisPass.class)) {
+      entries =
+          (List<DecodeEntry>) passResults.lastNullableResultOf(
+              VdtConstraintSynthesisPass.class);
+    } else {
+      entries =
+          (List<DecodeEntry>) passResults.lastNullableResultOf(VdtInputPreparationPass.class);
+    }
+    return entries != null ? entries.size() : 0;
+  }
+
   public static List<InfoEnricher> all = List.of(
       VDT_STATS_TAGS,
       VDT_STATS_EXPANDABLE,

vadl/main/vadl/pass/PassOrders.java

@@ -16,7 +16,7 @@
 
 package vadl.pass;
 
-import static vadl.configuration.DecoderOptions.Generator.RTL_TABLE;
+import static vadl.configuration.DecoderOptions.Generator.TABLE;
 import static vadl.configuration.DecoderOptions.OptionToSkip.OPT_CONSTRAINT_SYNTHESIS;
 import static vadl.configuration.DecoderOptions.OptionToSkip.OPT_DECODER_VERIFICATION;
 import static vadl.configuration.DecoderOptions.OptionToSkip.OPT_ENCODING_VERIFICATION;
@@ -699,8 +699,8 @@ public static PassOrder rtl(RtlConfiguration config) throws IOException {
         "mia-inline",
         "MiA after inlining instruction behavior");
 
-    if (config.getDecoderOptions().getGenerator() != RTL_TABLE) {
-      // Prepares and constructs the VDT, which is not used by the rtl-table strategy
+    if (config.getDecoderOptions().getGenerator() != TABLE) {
+      // Prepares and constructs the VDT, which is not used by the 'table' strategy
       addDecodePasses(order, config);
     }
 

vadl/main/vadl/vdt/impl/irregular/IrregularDecodeTreeGenerator.java

@@ -100,6 +100,10 @@ protected Node generateInternal(DecodeEntries decodeEntries) {
       return makeNode(decodeEntries);
     }
 
+    if (decodeEntries.isEmpty()) {
+      throw new IllegalStateException("Entry set must not be empty");
+    }
+
     // It's possible to have multiple decode entries pointing to the same instruction
     var entry = combineEntries(decodeEntries);
 
@@ -230,18 +234,31 @@ protected SingleSplitEntrySet split(DecodeEntries decodeEntries, BitPattern patt
 
     for (DecodeEntry e : decodeEntries.entries()) {
 
-      var matchesThen = compatible(e.pattern(), pattern) && e.exclusionConditions().stream()
-          .noneMatch(c -> contain(pattern, c.matching())
-              && c.unmatching().stream().noneMatch(p -> compatible(pattern, p)));
-      var matchesElse = !contain(e.pattern(), pattern);
-
-      if (matchesThen ^ matchesElse) {
+      // See if there is an encoding that matches the base pattern and is not excluded by the
+      // exclusion conditions
+      var mThen = compatible(e.pattern(), pattern) && e.exclusionConditions().stream()
+          .allMatch(ex -> {
+            var combined = combinePatterns(e.pattern(), pattern);
+            return !contain(combined, ex.matching()) || ex.unmatching().stream()
+                .anyMatch(u -> compatible(combined, u));
+          });
+
+      // See if there is an encoding that either does not match the base pattern or is excluded by
+      // the exclusion conditions (and not re-included by the unmatching patterns)
+      var mElse = !compatible(e.pattern(), pattern) || e.exclusionConditions().stream()
+          .anyMatch(ex -> {
+            var combined = combinePatterns(e.pattern(), pattern);
+            return compatible(combined, ex.matching()) && ex.unmatching().stream()
+                .noneMatch(u -> compatible(combined, u));
+          });
+
+      if (mThen ^ mElse) {
         // The entry occurs only on one side of the branch, we can keep the current probability.
-        (matchesThen ? matchingEntries : otherEntries).add(e);
+        (mThen ? matchingEntries : otherEntries).add(e);
         continue;
       }
 
-      if (!matchesThen) {
+      if (!mThen) {
         // Must not happen
         throw toConstructionDiagnostic(decodeEntries);
       }

vadl/main/vadl/vdt/impl/irregular/OccurrenceAwareDecodeTreeGenerator.java

@@ -148,9 +148,13 @@ private Optional<Pair<Double, MultiSplitEntrySet>> bestMultiDecision(
         .filter(Objects::nonNull)
         .min(Comparator.comparing(Pair::left)).orElse(null);
 
+    if (base == null) {
+      return Optional.empty();
+    }
+
     final var checked = decodeEntries.checkedBits().toMaskVector();
 
-    Pair<Double, MultiSplitEntrySet> prev = Objects.requireNonNull(base);
+    Pair<Double, MultiSplitEntrySet> prev = base;
     Pair<Double, MultiSplitEntrySet> next = prev;
     do {
 
@@ -224,7 +228,7 @@ private Optional<BitPattern> findBestPattern(final BitPattern base,
 
     } while (nextCost < prevCost);
 
-    return Optional.of(prev);
+    return prev == base ? Optional.empty() : Optional.of(prev);
   }
 
   /**

vadl/main/vadl/vdt/impl/irregular/model/DecodeEntries.java

@@ -33,4 +33,8 @@ public boolean hasMultiple() {
     return entries.stream().map(DecodeEntry::source).collect(Collectors.toSet()).size() > 1;
   }
 
+  public boolean isEmpty() {
+    return entries.isEmpty();
+  }
+
 }

vadl/main/vadl/vdt/passes/VdtInputPreparationPass.java

@@ -104,7 +104,7 @@ public Object execute(PassResults passResults, Specification viam) throws IOExce
           final var pattern = toFixedBitPattern(i, bo);
           final var exclusions = getExclusionsFromConstraints(bo, i);
 
-          // TODO: Add the option to supply occurrence probabilities. For now we assume uniform
+          // TODO: Process & parse the occurrence probabilities. For now we'll assume uniform
           //       distribution.
           final double probability = 1.0 / isa.ownInstructions().size();