test SQLTokenizedFormatter to separate the concerns

lib/src/main/java/com/diffplug/spotless/generic/IndentStep.java

@@ -102,14 +102,10 @@ String format(String raw) {
 				if (numSpaces > 0) {
 					switch (state.type) {
 					case SPACE:
-						for (int i = 0; i < numSpaces; ++i) {
-							builder.append(' ');
-						}
+						builder.append(" ".repeat(numSpaces));
 						break;
 					case TAB:
-						for (int i = 0; i < numSpaces / state.numSpacesPerTab; ++i) {
-							builder.append('\t');
-						}
+						builder.append("\t".repeat(Math.max(0, numSpaces / state.numSpacesPerTab)));
 						if (mightBeMultiLineComment && (numSpaces % state.numSpacesPerTab == 1)) {
 							builder.append(' ');
 						}

lib/src/main/java/com/diffplug/spotless/groovy/RemoveSemicolonsStep.java

@@ -22,6 +22,8 @@
 import com.diffplug.spotless.FormatterFunc;
 import com.diffplug.spotless.FormatterStep;
 
+import static java.lang.System.lineSeparator;
+
 /**
  * Removes all semicolons from the end of lines.
  *
@@ -51,7 +53,7 @@ FormatterFunc toFormatter() {
 					String line;
 					while ((line = reader.readLine()) != null) {
 						result.append(removeSemicolon(line));
-						result.append(System.lineSeparator());
+						result.append(lineSeparator());
 					}
 					return result.toString();
 				}

lib/src/main/java/com/diffplug/spotless/sql/dbeaver/DBeaverSQLFormatterConfiguration.java

@@ -67,9 +67,7 @@ public DBeaverSQLFormatterConfiguration(Properties properties) {
 	private String getIndentString(String indentType, int indentSize) {
 		char indentChar = indentType.equals("space") ? ' ' : '\t';
 		StringBuilder stringBuilder = new StringBuilder();
-		for (int i = 0; i < indentSize; i++) {
-			stringBuilder.append(indentChar);
-		}
+		stringBuilder.append(String.valueOf(indentChar).repeat(Math.max(0, indentSize)));
 		return stringBuilder.toString();
 	}
 

lib/src/main/java/com/diffplug/spotless/sql/dbeaver/SQLTokenizedFormatter.java

@@ -38,38 +38,24 @@ public class SQLTokenizedFormatter {
 
 	private static final String[] JOIN_BEGIN = {"LEFT", "RIGHT", "INNER", "OUTER", "JOIN"};
 	private static final SQLDialect sqlDialect = SQLDialect.INSTANCE;
-	private DBeaverSQLFormatterConfiguration formatterCfg;
-	private List<Boolean> functionBracket = new ArrayList<>();
-	private List<String> statementDelimiters = new ArrayList<>(2);
+	private final DBeaverSQLFormatterConfiguration formatterCfg;
+	private final List<Boolean> functionBracket = new ArrayList<>();
+	private final List<String> statementDelimiters = new ArrayList<>(2);
 
 	public SQLTokenizedFormatter(DBeaverSQLFormatterConfiguration formatterCfg) {
 		this.formatterCfg = formatterCfg;
 	}
 
 	public String format(final String argSql) {
-
 		statementDelimiters.add(formatterCfg.getStatementDelimiter());
-		SQLTokensParser fParser = new SQLTokensParser();
-
 		functionBracket.clear();
-
-		boolean isSqlEndsWithNewLine = false;
-		if (argSql.endsWith("\n")) {
-			isSqlEndsWithNewLine = true;
-		}
-
-		List<FormatterToken> list = fParser.parse(argSql);
-		list = format(list);
-
 		StringBuilder after = new StringBuilder(argSql.length() + 20);
-		for (FormatterToken token : list) {
+		for (FormatterToken token : format(new SQLTokensParser().parse(argSql))) {
 			after.append(token.getString());
 		}
-
-		if (isSqlEndsWithNewLine) {
+		if (argSql.endsWith("\n")) {
 			after.append(getDefaultLineSeparator());
 		}
-
 		return after.toString();
 	}
 
@@ -94,27 +80,31 @@ private List<FormatterToken> format(final List<FormatterToken> argList) {
 			}
 		}
 
-		final KeywordCase keywordCase = formatterCfg.getKeywordCase();
 		for (FormatterToken anArgList : argList) {
 			token = anArgList;
 			if (token.getType() == TokenType.KEYWORD) {
-				token.setString(keywordCase.transform(token.getString()));
+				token.setString(formatterCfg.getKeywordCase().transform(token.getString()));
 			}
 		}
 
-		// Remove extra tokens (spaces, etc)
-		for (int index = argList.size() - 1; index >= 1; index--) {
-			token = argList.get(index);
-			FormatterToken prevToken = argList.get(index - 1);
-			if (token.getType() == TokenType.SPACE && (prevToken.getType() == TokenType.SYMBOL || prevToken.getType() == TokenType.COMMENT)) {
-				argList.remove(index);
-			} else if ((token.getType() == TokenType.SYMBOL || token.getType() == TokenType.COMMENT) && prevToken.getType() == TokenType.SPACE) {
-				argList.remove(index - 1);
-			} else if (token.getType() == TokenType.SPACE) {
-				token.setString(" ");
-			}
-		}
+		Remove_extra_tokens(argList);
 
+		extracted2(argList);
+
+		int indent = 0;
+		final List<Integer> bracketIndent = new ArrayList<>();
+		FormatterToken prev = new FormatterToken(TokenType.SPACE, " ");
+		boolean encounterBetween = false;
+		extracted(argList, prev, bracketIndent, indent, encounterBetween);
+
+		extracted(argList);
+
+		extracted1(argList);
+
+		return argList;
+	}
+
+	private static void extracted2(List<FormatterToken> argList) {
 		for (int index = 0; index < argList.size() - 2; index++) {
 			FormatterToken t0 = argList.get(index);
 			FormatterToken t1 = argList.get(index + 1);
@@ -146,11 +136,90 @@ private List<FormatterToken> format(final List<FormatterToken> argList) {
 				argList.remove(index + 1);
 			}
 		}
+	}
 
-		int indent = 0;
-		final List<Integer> bracketIndent = new ArrayList<>();
-		FormatterToken prev = new FormatterToken(TokenType.SPACE, " ");
-		boolean encounterBetween = false;
+	private static void Remove_extra_tokens(List<FormatterToken> argList) {
+		FormatterToken token;
+		// Remove extra tokens (spaces, etc)
+		for (int index = argList.size() - 1; index >= 1; index--) {
+			token = argList.get(index);
+			FormatterToken prevToken = argList.get(index - 1);
+			if (token.getType() == TokenType.SPACE && (prevToken.getType() == TokenType.SYMBOL || prevToken.getType() == TokenType.COMMENT)) {
+				argList.remove(index);
+			} else if ((token.getType() == TokenType.SYMBOL || token.getType() == TokenType.COMMENT) && prevToken.getType() == TokenType.SPACE) {
+				argList.remove(index - 1);
+			} else if (token.getType() == TokenType.SPACE) {
+				token.setString(" ");
+			}
+		}
+	}
+
+	private void extracted1(List<FormatterToken> argList) {
+		FormatterToken prev;
+		FormatterToken token;
+		for (int index = 1; index < argList.size(); index++) {
+			prev = argList.get(index - 1);
+			token = argList.get(index);
+
+			if (prev.getType() != TokenType.SPACE &&
+					token.getType() != TokenType.SPACE &&
+					!token.getString().startsWith("(")) {
+				if (token.getString().equals(",") || statementDelimiters.contains(token.getString())) {
+					continue;
+				}
+				if (isFunction(prev.getString())
+						&& token.getString().equals("(")) {
+					continue;
+				}
+				if (token.getType() == TokenType.VALUE && prev.getType() == TokenType.NAME) {
+					// Do not add space between name and value [JDBC:MSSQL]
+					continue;
+				}
+				if (token.getType() == TokenType.SYMBOL && isEmbeddedToken(token) ||
+						prev.getType() == TokenType.SYMBOL && isEmbeddedToken(prev)) {
+					// Do not insert spaces around colons
+					continue;
+				}
+				if (token.getType() == TokenType.SYMBOL && prev.getType() == TokenType.SYMBOL) {
+					// Do not add space between symbols
+					continue;
+				}
+				if (prev.getType() == TokenType.COMMENT) {
+					// Do not add spaces to comments
+					continue;
+				}
+				argList.add(index, new FormatterToken(TokenType.SPACE, " "));
+			}
+		}
+	}
+
+	private static void extracted(List<FormatterToken> argList) {
+		for (int index = argList.size() - 1; index >= 4; index--) {
+			if (index >= argList.size()) {
+				continue;
+			}
+
+			FormatterToken t0 = argList.get(index);
+			FormatterToken t1 = argList.get(index - 1);
+			FormatterToken t2 = argList.get(index - 2);
+			FormatterToken t3 = argList.get(index - 3);
+			FormatterToken t4 = argList.get(index - 4);
+
+			if (t4.getString().equals("(")
+					&& t3.getString().trim().isEmpty()
+					&& t1.getString().trim().isEmpty()
+					&& t0.getString().equalsIgnoreCase(")")) {
+				t4.setString(t4.getString() + t2.getString() + t0.getString());
+				argList.remove(index);
+				argList.remove(index - 1);
+				argList.remove(index - 2);
+				argList.remove(index - 3);
+			}
+		}
+	}
+
+	private void extracted(List<FormatterToken> argList, FormatterToken prev, List<Integer> bracketIndent, int indent, boolean encounterBetween) {
+		FormatterToken token;
 		for (int index = 0; index < argList.size(); index++) {
 			token = argList.get(index);
 			String tokenString = token.getString().toUpperCase(Locale.ENGLISH);
@@ -280,66 +349,6 @@ private List<FormatterToken> format(final List<FormatterToken> argList) {
 			}
 			prev = token;
 		}
-
-		for (int index = argList.size() - 1; index >= 4; index--) {
-			if (index >= argList.size()) {
-				continue;
-			}
-
-			FormatterToken t0 = argList.get(index);
-			FormatterToken t1 = argList.get(index - 1);
-			FormatterToken t2 = argList.get(index - 2);
-			FormatterToken t3 = argList.get(index - 3);
-			FormatterToken t4 = argList.get(index - 4);
-
-			if (t4.getString().equals("(")
-					&& t3.getString().trim().isEmpty()
-					&& t1.getString().trim().isEmpty()
-					&& t0.getString().equalsIgnoreCase(")")) {
-				t4.setString(t4.getString() + t2.getString() + t0.getString());
-				argList.remove(index);
-				argList.remove(index - 1);
-				argList.remove(index - 2);
-				argList.remove(index - 3);
-			}
-		}
-
-		for (int index = 1; index < argList.size(); index++) {
-			prev = argList.get(index - 1);
-			token = argList.get(index);
-
-			if (prev.getType() != TokenType.SPACE &&
-					token.getType() != TokenType.SPACE &&
-					!token.getString().startsWith("(")) {
-				if (token.getString().equals(",") || statementDelimiters.contains(token.getString())) {
-					continue;
-				}
-				if (isFunction(prev.getString())
-						&& token.getString().equals("(")) {
-					continue;
-				}
-				if (token.getType() == TokenType.VALUE && prev.getType() == TokenType.NAME) {
-					// Do not add space between name and value [JDBC:MSSQL]
-					continue;
-				}
-				if (token.getType() == TokenType.SYMBOL && isEmbeddedToken(token) ||
-						prev.getType() == TokenType.SYMBOL && isEmbeddedToken(prev)) {
-					// Do not insert spaces around colons
-					continue;
-				}
-				if (token.getType() == TokenType.SYMBOL && prev.getType() == TokenType.SYMBOL) {
-					// Do not add space between symbols
-					continue;
-				}
-				if (prev.getType() == TokenType.COMMENT) {
-					// Do not add spaces to comments
-					continue;
-				}
-				argList.add(index, new FormatterToken(TokenType.SPACE, " "));
-			}
-		}
-
-		return argList;
 	}
 
 	private static boolean isEmbeddedToken(FormatterToken token) {
@@ -351,7 +360,7 @@ private boolean isJoinStart(List<FormatterToken> argList, int index) {
 		// And we must be in the beginning of sequence
 
 		// check current token
-		if (!contains(JOIN_BEGIN, argList.get(index).getString())) {
+		if (!contains(argList.get(index).getString())) {
 			return false;
 		}
 		// check previous token
@@ -360,7 +369,7 @@ private boolean isJoinStart(List<FormatterToken> argList, int index) {
 			if (token.getType() == TokenType.SPACE) {
 				continue;
 			}
-			if (contains(JOIN_BEGIN, token.getString())) {
+			if (contains(token.getString())) {
 				// It is not the begin of sequence
 				return false;
 			} else {
@@ -376,7 +385,7 @@ private boolean isJoinStart(List<FormatterToken> argList, int index) {
 			if (token.getString().equals("JOIN")) {
 				return true;
 			}
-			if (!contains(JOIN_BEGIN, token.getString())) {
+			if (!contains(token.getString())) {
 				// It is not the begin of sequence
 				return false;
 			}
@@ -398,14 +407,12 @@ private int insertReturnAndIndent(final List<FormatterToken> argList, final int
 		try {
 			final String defaultLineSeparator = getDefaultLineSeparator();
 			StringBuilder s = new StringBuilder(defaultLineSeparator);
-			for (int index = 0; index < argIndent; index++) {
-				s.append(formatterCfg.getIndentString());
-			}
+			s.append(String.valueOf(formatterCfg.getIndentString()).repeat(Math.max(0, argIndent)));
 			if (argIndex > 0) {
 				final FormatterToken token = argList.get(argIndex);
 				final FormatterToken prevToken = argList.get(argIndex - 1);
 				if (token.getType() == TokenType.COMMENT &&
-						isCommentLine(sqlDialect, token.getString()) &&
+						isCommentLine(token.getString()) &&
 						prevToken.getType() != TokenType.END) {
 					s.setCharAt(0, ' ');
 					s.setLength(1);
@@ -446,19 +453,19 @@ private int insertReturnAndIndent(final List<FormatterToken> argList, final int
 		}
 	}
 
-	private static boolean isCommentLine(SQLDialect dialect, String line) {
-		for (String slc : dialect.getSingleLineComments()) {
+	private static boolean isCommentLine(String line) {
+		for (String slc : SQLTokenizedFormatter.sqlDialect.getSingleLineComments()) {
 			if (line.startsWith(slc)) {
 				return true;
 			}
 		}
 		return false;
 	}
 
-	private static <OBJECT_TYPE> boolean contains(OBJECT_TYPE[] array, OBJECT_TYPE value) {
-		if (array == null || array.length == 0)
+	private static <OBJECT_TYPE> boolean contains(OBJECT_TYPE value) {
+		if (SQLTokenizedFormatter.JOIN_BEGIN == null)
 			return false;
-		for (OBJECT_TYPE anArray : array) {
+		for (OBJECT_TYPE anArray : (OBJECT_TYPE[]) SQLTokenizedFormatter.JOIN_BEGIN) {
 			if (Objects.equals(value, anArray))
 				return true;
 		}

lib/src/main/java/com/diffplug/spotless/sql/dbeaver/SQLTokensParser.java

@@ -292,7 +292,7 @@ List<FormatterToken> parse(final String argSql) {
 	}
 
 	private static boolean contains(char[] array, char value) {
-		if (array == null || array.length == 0)
+		if (array == null)
 			return false;
 		for (char aChar : array) {
 			if (aChar == value)