Explain files, Add todos, remove unused stuff

MightyHelper · MightyHelper · commit 74716b7fee66 · 2023-04-08T21:18:05.000-03:00
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
diff --git a/src/main/java/ko/carbonel/Logger.java b/src/main/java/ko/carbonel/Logger.java
@@ -33,9 +33,4 @@ public static void internalTrace(Throwable e) {
 			e.printStackTrace();
 		}
 	}
-
-	public static String getSymbol(char s) {
-		if (s == Utils.EOF) return "[EOF]";
-		return String.valueOf(s);
-	}
 }
diff --git a/src/main/java/ko/carbonel/Utils.java b/src/main/java/ko/carbonel/Utils.java
@@ -17,6 +17,7 @@
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
+// TODO: Consider more coverage
 public class Utils {
 	public static final char EOF = 0xffff;
 	public static final String STR_EOF = String.valueOf(Utils.EOF);
diff --git a/src/main/java/ko/carbonel/compiler/config/Config.java b/src/main/java/ko/carbonel/compiler/config/Config.java
@@ -12,8 +12,8 @@
 import ko.carbonel.compiler.runner.output.PrettyTableWriter;
 import ko.carbonel.compiler.runner.output.TableWriter;
 import ko.carbonel.syntax.grammar.generator.AlphabeticRuleSorter;
+import ko.carbonel.syntax.grammar.generator.RuleSorter;
 import ko.carbonel.syntax.grammar.generator.StartAwareRuleSorter;
-import ko.carbonel.syntax.grammar.model.Rule;
 import ko.carbonel.syntax.grammar.model.Rules;
 
 import java.io.File;
@@ -124,7 +124,7 @@ private String getSortFun() {
 		return params.get(SORT_FUN);
 	}
 
-	public Comparator<? super Rule> getActiveSorter(Rules rules) {
+	public RuleSorter getActiveSorter(Rules rules) {
 		// We will use rules parameter in the future if we want to do a dfs or bfs sort
 		if (Objects.equals(getSortFun(), ALPHABETIC_SORT)) return new AlphabeticRuleSorter();
 		return new StartAwareRuleSorter();
diff --git a/src/main/java/ko/carbonel/compiler/generated/Parser.java b/src/main/java/ko/carbonel/compiler/generated/Parser.java
diff --git a/src/main/java/ko/carbonel/syntax/grammar/algorithm/BNFConverter.java b/src/main/java/ko/carbonel/syntax/grammar/algorithm/BNFConverter.java
@@ -14,7 +14,7 @@
 
 public class BNFConverter {
 	public static Rules convertBNF(Rules r) {
-		while (r.rules().stream().anyMatch(rule -> rule.options().stream().anyMatch(to -> to.tokens().stream().anyMatch(Token::isBNFE)))) {
+		while (r.rules().stream().anyMatch(rule -> rule.options().stream().anyMatch(To::isBNFE))) {
 			Stream<Rule> newRules = r.rules().stream().flatMap(BNFConverter::convertBNF);
 			r = new Rules(newRules.toList()).simplify();
 		}
diff --git a/src/main/java/ko/carbonel/syntax/grammar/algorithm/DeFactorizer.java b/src/main/java/ko/carbonel/syntax/grammar/algorithm/DeFactorizer.java
@@ -7,12 +7,9 @@
 
 public class DeFactorizer {
 	public static Rules deFactorize(Rules r) {
-		// Find all rules that are only used once
 		Rules finalR = r;
 		List<Token> nonTerminalTokens = r.getAllTokens().stream().filter(Token::isNonTerminal).toList();
-//		List<Token> toDelete = nonTerminalTokens.stream().filter(token -> finalR.getRuleSectionsThatProduce(token).size() == 0).toList();
 		List<Token> toReplace = nonTerminalTokens.stream().filter(token -> finalR.getRuleSectionsThatProduce(token).size() == 1).toList();
-//		List<Token> toLeaveIntact = nonTerminalTokens.stream().filter(token -> finalR.getRuleSectionsThatProduce(token).size() > 1).toList();
 		for (Token token : toReplace) {
 			r = r.replaceToken(token, r.getRuleFor(token).options());
 		}
diff --git a/src/main/java/ko/carbonel/syntax/grammar/algorithm/Factorizer.java b/src/main/java/ko/carbonel/syntax/grammar/algorithm/Factorizer.java
@@ -11,6 +11,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+// TODO: Add coverage
 public class Factorizer {
 	public static final String FACTORED_EXTRA = "F";
 	static Map<Rule, List<Token>> multiPrefixRules = new HashMap<>();
diff --git a/src/main/java/ko/carbonel/syntax/grammar/algorithm/FirstCalculator.java b/src/main/java/ko/carbonel/syntax/grammar/algorithm/FirstCalculator.java
@@ -9,6 +9,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+// TODO: Add coverage
 public class FirstCalculator {
 	public static Set<Token> calculateFirst(List<Token> gamma, Rules r) {
 		if (gamma.size() == 0) throw new IllegalArgumentException("Cannot calculate first for empty list " + r);
@@ -25,7 +26,7 @@ public static Set<Token> calculateFullFirst(Token from, To to, Rules grammar) {
 		Set<Token> first = FirstCalculator.calculateFirst(to.tokens(), grammar);
 		if (first.contains(Token.LAMBDA)) {
 			first.remove(Token.LAMBDA);
-			first.addAll(NextCalculator.calculateNext2(from, grammar));
+			first.addAll(NextCalculator.calculateNext(from, grammar));
 		}
 		return first;
 	}
diff --git a/src/main/java/ko/carbonel/syntax/grammar/algorithm/NextCalculator.java b/src/main/java/ko/carbonel/syntax/grammar/algorithm/NextCalculator.java
@@ -1,35 +1,16 @@
 package ko.carbonel.syntax.grammar.algorithm;
 
 import ko.carbonel.Pair;
-import ko.carbonel.Utils;
 import ko.carbonel.syntax.ParserGenerator;
-import ko.carbonel.syntax.grammar.model.Rule;
 import ko.carbonel.syntax.grammar.model.Rules;
 import ko.carbonel.syntax.grammar.model.Token;
 
 import java.util.*;
 import java.util.function.BiFunction;
 import java.util.stream.IntStream;
 
+// TODO: Add coverage
 public class NextCalculator {
-	// Not working
-	public static Set<Token> calculateNext(Token gamma, Rules r) {
-		List<Rule> rulesThatProduce = r.getRulesThatProduce(gamma);
-		Set<Token> out = new HashSet<>();
-		rulesThatProduce.forEach(rule -> rule.options().forEach(to -> {
-			if (!to.tokens().contains(gamma)) return;
-			List<Token> betas = Utils.indicesOf(gamma, to.tokens()).mapToObj(i -> to.tokens().get(i)).toList();
-			if (betas.isEmpty()) {
-				out.addAll(calculateNext(rule.from(), r));
-			}
-			betas.forEach(beta -> {
-				if (beta.equals(Token.LAMBDA)) out.addAll(calculateNext(rule.from(), r));
-				else out.addAll(FirstCalculator.calculateFirst(List.of(rule.from()), r));
-			});
-		}));
-		return out;
-	}
-
 	private static <T> void streamInPairs(List<T> list, BiFunction<Pair<T, T>, Boolean, Void> fun) {
 		if (list.size() == 1) {
 			fun.apply(new Pair<>(null, list.get(0)), true);
@@ -38,7 +19,10 @@ private static <T> void streamInPairs(List<T> list, BiFunction<Pair<T, T>, Boole
 		}
 	}
 
-	public static Set<Token> calculateNext2(Token gamma, Rules r) {
+	// TODO: Clean this up
+	// TODO: Add Caching
+	// TODO: Avoid calculating all follows for each token everytime
+	public static Set<Token> calculateNext(Token gamma, Rules r) {
 		// We should probably cache this
 		Map<Token, Set<Token>> follows = new HashMap<>();
 		store(follows, ParserGenerator.START, Token.EOF);
diff --git a/src/main/java/ko/carbonel/syntax/grammar/algorithm/RecursionRemover.java b/src/main/java/ko/carbonel/syntax/grammar/algorithm/RecursionRemover.java
@@ -13,6 +13,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+// TODO: Further coverage
+// TODO: Split up functions
 public class RecursionRemover {
 	public static final String RECURSION_REMOVED_EXTRA = "R";
 
@@ -26,6 +28,7 @@ public static Boolean hasIndirectRecursion(Token t, Rules r) {
 			} while (lastSize != active.size() && !active.contains(t));
 			return active.contains(t);
 		} catch (Exception e) {
+			// Used to avoid some issues when displaying tables
 			return null;
 		}
 	}
diff --git a/src/main/java/ko/carbonel/syntax/grammar/algorithm/SameProductionRemover.java b/src/main/java/ko/carbonel/syntax/grammar/algorithm/SameProductionRemover.java
@@ -8,6 +8,7 @@
 import java.util.Comparator;
 import java.util.List;
 
+// TODO: Add coverage
 public class SameProductionRemover {
 	// Sometimes after factorizing we get A -> a b, and B -> a b, so we remove the duplicates
 	// We do this twice because sometimes we get A -> a C b, B -> a D b, C -> a b, and D -> a b
diff --git a/src/main/java/ko/carbonel/syntax/grammar/generator/AlphabeticRuleSorter.java b/src/main/java/ko/carbonel/syntax/grammar/generator/AlphabeticRuleSorter.java
@@ -2,6 +2,9 @@
 
 import ko.carbonel.syntax.grammar.model.Rule;
 
+/*
+	* Sorts rules alphabetically, and disregards the start rule
+*/
 public class AlphabeticRuleSorter implements RuleSorter {
 	@Override
 	public int compare(Rule o1, Rule o2) {
diff --git a/src/main/java/ko/carbonel/syntax/grammar/generator/BNFFormatter.java b/src/main/java/ko/carbonel/syntax/grammar/generator/BNFFormatter.java
@@ -2,6 +2,9 @@
 
 import ko.carbonel.syntax.grammar.model.Rules;
 
+/*
+ * This interface is used to format a grammar into a string.
+ */
 public interface BNFFormatter {
 	String format(Rules r);
 }
diff --git a/src/main/java/ko/carbonel/syntax/grammar/generator/MyBNFFormatter.java b/src/main/java/ko/carbonel/syntax/grammar/generator/MyBNFFormatter.java
@@ -8,6 +8,9 @@
 
 import java.util.stream.Collectors;
 
+/**
+ * Generates .syn format from a grammar.
+ */
 public class MyBNFFormatter implements BNFFormatter {
 	@Override
 	public String format(Rules rules) {
diff --git a/src/main/java/ko/carbonel/syntax/grammar/generator/RuleCodeGenerator.java b/src/main/java/ko/carbonel/syntax/grammar/generator/RuleCodeGenerator.java
@@ -156,7 +156,7 @@ private String getValidOptions(Rule rule) {
 					Set<Token> first = FirstCalculator.calculateFullFirst(rule.from(), to, grammar);
 					return first.stream();
 				}
-				Set<Token> next = NextCalculator.calculateNext2(rule.from(), grammar);
+				Set<Token> next = NextCalculator.calculateNext(rule.from(), grammar);
 				return next.stream();
 			}).distinct()
 			.map(token -> rewriteTerminalName(token.value()))
@@ -174,7 +174,7 @@ private String generateCode(Token from, To to, boolean doElse) {
 				generateRuleCondition(first) +
 				"){\n" + generateMatcherCode(to) + "\n\t\t}";
 		}
-		Set<Token> next = NextCalculator.calculateNext2(from, grammar);
+		Set<Token> next = NextCalculator.calculateNext(from, grammar);
 		return (doElse ? "else " : "") + (doElse ? "" : "\t\t") + "//noinspection StatementWithEmptyBody\n\t\t" + (doElse ? " " : "") + "if (" +
 			generateRuleCondition(next) +
 			"){\n" + generateMatcherCode(to) + "\n\t\t}";
diff --git a/src/main/java/ko/carbonel/syntax/grammar/generator/RuleSorter.java b/src/main/java/ko/carbonel/syntax/grammar/generator/RuleSorter.java
@@ -4,5 +4,8 @@
 
 import java.util.Comparator;
 
+/*
+ * This interface is used to sort rules in several locations
+ */
 public interface RuleSorter extends Comparator<Rule> {
 }
diff --git a/src/main/java/ko/carbonel/syntax/grammar/generator/RuleTable.java b/src/main/java/ko/carbonel/syntax/grammar/generator/RuleTable.java
@@ -17,6 +17,9 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+/*
+ * An unused implementation of a Table-based parser.
+ */
 public class RuleTable {
 	Map<Pair<Token, Token>, Set<To>> table = new HashMap<>();
 	Set<Token> collisions = new HashSet<>();
@@ -28,7 +31,7 @@ public class RuleTable {
 			rule.options().forEach(to -> FirstCalculator.calculateFirst(to.tokens(), r).forEach(a -> {
 				insert(A, a, to);
 				Set<Token> toFirst = FirstCalculator.calculateFirst(to.tokens(), r);
-				Set<Token> ANext = NextCalculator.calculateNext2(A, r);
+				Set<Token> ANext = NextCalculator.calculateNext(A, r);
 				if (toFirst.contains(Token.LAMBDA)) {
 					ANext.forEach(b -> insert(A, b, to));
 				}
diff --git a/src/main/java/ko/carbonel/syntax/grammar/generator/StandardBNFFormatter.java b/src/main/java/ko/carbonel/syntax/grammar/generator/StandardBNFFormatter.java
@@ -9,6 +9,9 @@
 
 import java.util.stream.Collectors;
 
+/*
+ * Generates standardised .bnf format from a grammar.
+ */
 public class StandardBNFFormatter implements BNFFormatter {
 	@Override
 	public String format(Rules r) {
diff --git a/src/main/java/ko/carbonel/syntax/grammar/generator/StartAwareRuleSorter.java b/src/main/java/ko/carbonel/syntax/grammar/generator/StartAwareRuleSorter.java
@@ -3,6 +3,9 @@
 import ko.carbonel.syntax.ParserGenerator;
 import ko.carbonel.syntax.grammar.model.Rule;
 
+/*
+ * Sorts rules alphabetically, and puts the start rule first
+ */
 public class StartAwareRuleSorter implements RuleSorter {
 	private static boolean isStart(Rule o2) {
 		return o2.from().equals(ParserGenerator.START);
diff --git a/src/main/java/ko/carbonel/syntax/grammar/model/Rule.java b/src/main/java/ko/carbonel/syntax/grammar/model/Rule.java
@@ -7,7 +7,7 @@
 import java.util.stream.Collectors;
 
 /**
- * This class is still under development and will be used for manipulating the tinyRust grammar
+ * A single Rule, with a single left-hand side and multiple right-hand sides.
  */
 public record Rule(Token from, List<To> options) {
 	public Rule(String from, List<To> options) {
diff --git a/src/main/java/ko/carbonel/syntax/grammar/model/Rules.java b/src/main/java/ko/carbonel/syntax/grammar/model/Rules.java
@@ -15,7 +15,7 @@
 import java.util.stream.Stream;
 
 /**
- * This class is still under development and will be used for manipulating the tinyRust grammar
+ * A list of rules, effectively a Grammar.
  */
 public class Rules {
 	final List<Rule> rules;
@@ -80,7 +80,7 @@ public List<List<String>> toTable(boolean first, boolean next, boolean ambiguous
 	}
 
 	private Set<Token> calculateNext(Rule it) {
-		return NextCalculator.calculateNext2(it.from(), this);
+		return NextCalculator.calculateNext(it.from(), this);
 	}
 
 	private Set<Token> calculateFirst(Rule it) {
@@ -153,10 +153,6 @@ public List<Rule> getRulesThatProduce(Token token) {
 		return rules.stream().filter(rule -> rule.options().stream().anyMatch(to -> to.tokens().contains(token))).toList();
 	}
 
-	public List<Rule> getRulesThatOnlyProduce(Token token) {
-		return rules.stream().filter(rule -> rule.options().stream().anyMatch(to -> to.onlyProduces(token))).toList();
-	}
-
 	public List<Rule> getRuleSectionsThatProduce(Token token) {
 		return getRulesThatProduce(token).stream()
 			.map(rule -> new Rule(
diff --git a/src/main/java/ko/carbonel/syntax/grammar/model/To.java b/src/main/java/ko/carbonel/syntax/grammar/model/To.java
@@ -1,20 +1,15 @@
 package ko.carbonel.syntax.grammar.model;
 
-import ko.carbonel.Pair;
-import ko.carbonel.Utils;
-import ko.carbonel.compiler.exception.TODO;
-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 import java.util.function.Function;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 /**
- * This class is still under development and will be used for manipulating the tinyRust grammar
+ * A single right-hand side of a Rule.
  */
 public record To(List<Token> tokens) {
 	public static final To LAMBDA = new To(List.of(Token.LAMBDA));
@@ -40,24 +35,6 @@ public boolean isBNFE() {
 		return tokens.stream().anyMatch(Token::isBNFE);
 	}
 
-	public Stream<Pair<String, String>> newRulesForKleene() {
-		return tokens.stream().flatMap(Token::newNameForKleene);
-	}
-
-	public Stream<To> rewriteOptional() {
-		List<Token> nonTerminals1 = tokens.stream().filter(Token::hasOptional).toList();
-		if (nonTerminals1.size() > 1) throw new TODO("More than one optional per target");
-		if (nonTerminals1.size() == 0) return Stream.of(new To(tokens));
-		return Stream.of(
-			new To(tokens.stream().map(Token::removeOptional).toList()), // has
-			new To(tokens.stream().map(Token::deleteOptional).filter(Objects::nonNull).toList()) // doesnt have
-		);
-	}
-
-	public To rewriteKleene() {
-		return new To(tokens.stream().map(Token::rewriteKleene).toList());
-	}
-
 	public Token optFirstToken() {
 		return tokens.isEmpty() ? null : tokens.get(0);
 	}
@@ -67,25 +44,12 @@ public String toString() {
 		return tokens.stream().map(Token::toString).collect(Collectors.joining(" "));
 	}
 
-	public To removeLambdas() {
-		if (tokens.size() <= 1) return new To(this.tokens());
-		return new To(tokens.stream().filter(it -> !Objects.equals(it, Utils.NT_LAMBDA)).toList());
-	}
-
-	public To removeTokens(Predicate<Token> predicate) {
-		return new To(tokens.stream().filter(predicate).toList());
-	}
-
 	public To removeNthToken(int n) {
 		List<Token> newTokens = new ArrayList<>(tokens);
 		newTokens.remove(n);
 		return new To(newTokens);
 	}
 
-	public To rewriteTokens(Function<Token, Token> rewrite) {
-		return new To(tokens.stream().map(rewrite).toList());
-	}
-
 	public To rewriteNthToken(int n, Function<Token, Token> rewrite) {
 		List<Token> newTokens = new ArrayList<>(tokens);
 		newTokens.set(n, rewrite.apply(newTokens.get(n)));
diff --git a/src/main/java/ko/carbonel/syntax/grammar/model/Token.java b/src/main/java/ko/carbonel/syntax/grammar/model/Token.java
@@ -3,10 +3,8 @@
 import ko.carbonel.Pair;
 import ko.carbonel.Utils;
 
-import java.util.stream.Stream;
-
 /**
- * This class is still under development and will be used for manipulating the tinyRust grammar
+ * A terminal or non-terminal token.
  */
 public record Token(String value) {
 	public static final Token LAMBDA = new Token(Utils.LAMBDA);
@@ -32,20 +30,12 @@ public Token removeOptional() {
 		return new Token(hasOptional() ? value.substring(0, value.length() - 1) : value);
 	}
 
-	public Token deleteOptional() {
-		return hasOptional() ? null : new Token(value);
-	}
-
 	public Token rewriteKleene() {
 		return hasKleene() ?
 		       new Token(getKleeneName()) :
 		       new Token(value);
 	}
 
-	public Stream<Pair<String, String>> newNameForKleene() {
-		return hasKleene() ? Stream.of(new Pair<>(value.replaceAll(Utils.KLEENE_STAR, ""), getKleeneName())) : Stream.empty();
-	}
-
 	public Pair<Token, Token> newNameForKleene2() {
 		return new Pair<>(new Token(value.replaceAll(Utils.KLEENE_STAR, "")), new Token(getKleeneName()));
 	}
diff --git a/src/main/java/ko/carbonel/syntax/grammar/productions/GrammarProducer.java b/src/main/java/ko/carbonel/syntax/grammar/productions/GrammarProducer.java
diff --git a/src/main/java/ko/carbonel/syntax/grammar/productions/KeyboardUI.java b/src/main/java/ko/carbonel/syntax/grammar/productions/KeyboardUI.java
diff --git a/src/main/java/ko/carbonel/syntax/grammar/productions/ProductionUI.java b/src/main/java/ko/carbonel/syntax/grammar/productions/ProductionUI.java
diff --git a/src/main/java/ko/carbonel/syntax/grammar/productions/RandomGenerator.java b/src/main/java/ko/carbonel/syntax/grammar/productions/RandomGenerator.java
diff --git a/src/test/java/ko/carbonel/compiler/generated/ParserTest.java b/src/test/java/ko/carbonel/compiler/generated/ParserTest.java

Original file line number	Diff line number	Diff line change
`@@ -33,9 +33,4 @@ public static void internalTrace(Throwable e) {`
`33`	`33`	`e.printStackTrace();`
`34`	`34`	`}`
`35`	`35`	`}`
`36`		`-`
`37`		`- public static String getSymbol(char s) {`
`38`		`- if (s == Utils.EOF) return "[EOF]";`
`39`		`- return String.valueOf(s);`
`40`		`- }`
`41`	`36`	`}`
Original file line number	Diff line number	Diff line change
`@@ -14,7 +14,7 @@`
`14`	`14`
`15`	`15`	`public class BNFConverter {`
`16`	`16`	`public static Rules convertBNF(Rules r) {`
`17`		`- while (r.rules().stream().anyMatch(rule -> rule.options().stream().anyMatch(to -> to.tokens().stream().anyMatch(Token::isBNFE)))) {`
	`17`	`+ while (r.rules().stream().anyMatch(rule -> rule.options().stream().anyMatch(To::isBNFE))) {`
`18`	`18`	`Stream<Rule> newRules = r.rules().stream().flatMap(BNFConverter::convertBNF);`
`19`	`19`	`r = new Rules(newRules.toList()).simplify();`
`20`	`20`	`}`