Move whitespaceRequiredWithinTokens from LegacyOverrides to BuiltInFeatures

jasmith-hs · jasmith-hs · commit 8678a9075986 · 2026-03-03T16:44:17.000-05:00
Removes the legacy override flag in favor of a feature flag, and migrates
parseWhitespaceControlStrictly to pass a WhitespaceControlParser enum through
the token pipeline rather than checking config at parse time.
diff --git a/src/main/java/com/hubspot/jinjava/LegacyOverrides.java b/src/main/java/com/hubspot/jinjava/LegacyOverrides.java
@@ -22,7 +22,6 @@ public interface LegacyOverrides extends WithLegacyOverrides {
     .withIterateOverMapKeys(true)
     .withUsePyishObjectMapper(true)
     .withUseSnakeCasePropertyNaming(true)
-    .withWhitespaceRequiredWithinTokens(true)
     .withUseNaturalOperatorPrecedence(true)
     .withParseWhitespaceControlStrictly(true)
     .withAllowAdjacentTextNodes(true)
@@ -50,11 +49,6 @@ default boolean isUseSnakeCasePropertyNaming() {
     return false;
   }
 
-  @Value.Default
-  default boolean isWhitespaceRequiredWithinTokens() {
-    return false;
-  }
-
   @Value.Default
   default boolean isUseNaturalOperatorPrecedence() {
     return false;
diff --git a/src/main/java/com/hubspot/jinjava/features/BuiltInFeatures.java b/src/main/java/com/hubspot/jinjava/features/BuiltInFeatures.java
@@ -1,6 +1,7 @@
 package com.hubspot.jinjava.features;
 
 public interface BuiltInFeatures {
+  String WHITESPACE_REQUIRED_WITHIN_TOKENS = "whitespace_required_within_tokens";
   String FIXED_DATE_TIME_FILTER_NULL_ARG = "FIXED_DATE_TIME_FILTER_NULL_ARG";
   String ECHO_UNDEFINED = "echoUndefined";
   String PREVENT_ACCIDENTAL_EXPRESSIONS = "PREVENT_ACCIDENTAL_EXPRESSIONS";
diff --git a/src/main/java/com/hubspot/jinjava/tree/TreeParser.java b/src/main/java/com/hubspot/jinjava/tree/TreeParser.java
@@ -38,13 +38,15 @@
 import com.hubspot.jinjava.tree.parse.TokenScanner;
 import com.hubspot.jinjava.tree.parse.TokenScannerSymbols;
 import com.hubspot.jinjava.tree.parse.UnclosedToken;
+import com.hubspot.jinjava.tree.parse.WhitespaceControlParser;
 import org.apache.commons.lang3.StringUtils;
 
 public class TreeParser {
 
   private final PeekingIterator<Token> scanner;
   private final JinjavaInterpreter interpreter;
   private final TokenScannerSymbols symbols;
+  private final WhitespaceControlParser whitespaceControlParser;
 
   private Node parent;
 
@@ -53,6 +55,10 @@ public TreeParser(JinjavaInterpreter interpreter, String input) {
       Iterators.peekingIterator(new TokenScanner(input, interpreter.getConfig()));
     this.interpreter = interpreter;
     this.symbols = interpreter.getConfig().getTokenScannerSymbols();
+    this.whitespaceControlParser =
+      interpreter.getConfig().getLegacyOverrides().isParseWhitespaceControlStrictly()
+        ? WhitespaceControlParser.STRICT
+        : WhitespaceControlParser.LENIENT;
   }
 
   public Node buildTree() {
@@ -178,7 +184,8 @@ private Node text(TextToken textToken) {
               StringUtils.stripEnd(textToken.getImage(), "\t "),
               textToken.getLineNumber(),
               textToken.getStartPosition(),
-              symbols
+              symbols,
+              whitespaceControlParser
             );
         }
       }
diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/ExpressionToken.java b/src/main/java/com/hubspot/jinjava/tree/parse/ExpressionToken.java
@@ -29,7 +29,17 @@ public ExpressionToken(
     int startPosition,
     TokenScannerSymbols symbols
   ) {
-    super(image, lineNumber, startPosition, symbols);
+    this(image, lineNumber, startPosition, symbols, WhitespaceControlParser.LENIENT);
+  }
+
+  public ExpressionToken(
+    String image,
+    int lineNumber,
+    int startPosition,
+    TokenScannerSymbols symbols,
+    WhitespaceControlParser whitespaceControlParser
+  ) {
+    super(image, lineNumber, startPosition, symbols, whitespaceControlParser);
   }
 
   @Override
diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/NoteToken.java b/src/main/java/com/hubspot/jinjava/tree/parse/NoteToken.java
@@ -25,7 +25,17 @@ public NoteToken(
     int startPosition,
     TokenScannerSymbols symbols
   ) {
-    super(image, lineNumber, startPosition, symbols);
+    this(image, lineNumber, startPosition, symbols, WhitespaceControlParser.LENIENT);
+  }
+
+  public NoteToken(
+    String image,
+    int lineNumber,
+    int startPosition,
+    TokenScannerSymbols symbols,
+    WhitespaceControlParser whitespaceControlParser
+  ) {
+    super(image, lineNumber, startPosition, symbols, whitespaceControlParser);
   }
 
   @Override
diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/TagToken.java b/src/main/java/com/hubspot/jinjava/tree/parse/TagToken.java
@@ -31,7 +31,17 @@ public TagToken(
     int startPosition,
     TokenScannerSymbols symbols
   ) {
-    super(image, lineNumber, startPosition, symbols);
+    this(image, lineNumber, startPosition, symbols, WhitespaceControlParser.LENIENT);
+  }
+
+  public TagToken(
+    String image,
+    int lineNumber,
+    int startPosition,
+    TokenScannerSymbols symbols,
+    WhitespaceControlParser whitespaceControlParser
+  ) {
+    super(image, lineNumber, startPosition, symbols, whitespaceControlParser);
   }
 
   @Override
diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/TextToken.java b/src/main/java/com/hubspot/jinjava/tree/parse/TextToken.java
@@ -27,7 +27,17 @@ public TextToken(
     int startPosition,
     TokenScannerSymbols symbols
   ) {
-    super(image, lineNumber, startPosition, symbols);
+    this(image, lineNumber, startPosition, symbols, WhitespaceControlParser.LENIENT);
+  }
+
+  public TextToken(
+    String image,
+    int lineNumber,
+    int startPosition,
+    TokenScannerSymbols symbols,
+    WhitespaceControlParser whitespaceControlParser
+  ) {
+    super(image, lineNumber, startPosition, symbols, whitespaceControlParser);
   }
 
   public void mergeImageAndContent(TextToken otherToken) {
diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/Token.java b/src/main/java/com/hubspot/jinjava/tree/parse/Token.java
@@ -15,9 +15,6 @@
  **********************************************************************/
 package com.hubspot.jinjava.tree.parse;
 
-import com.hubspot.jinjava.JinjavaConfig;
-import com.hubspot.jinjava.LegacyOverrides;
-import com.hubspot.jinjava.interpret.JinjavaInterpreter;
 import com.hubspot.jinjava.interpret.UnexpectedTokenException;
 import java.io.Serializable;
 
@@ -32,6 +29,7 @@ public abstract class Token implements Serializable {
   protected final int lineNumber;
   protected final int startPosition;
   private final TokenScannerSymbols symbols;
+  private final WhitespaceControlParser whitespaceControlParser;
 
   private boolean leftTrim;
   private boolean rightTrim;
@@ -41,12 +39,14 @@ public Token(
     String image,
     int lineNumber,
     int startPosition,
-    TokenScannerSymbols symbols
+    TokenScannerSymbols symbols,
+    WhitespaceControlParser whitespaceControlParser
   ) {
     this.image = image;
     this.lineNumber = lineNumber;
     this.startPosition = startPosition;
     this.symbols = symbols;
+    this.whitespaceControlParser = whitespaceControlParser;
     parse();
   }
 
@@ -89,25 +89,14 @@ public void setRightTrimAfterEnd(boolean rightTrimAfterEnd) {
    * @return the content stripped of any whitespace control characters.
    */
   protected final String handleTrim(String unwrapped) {
-    boolean parseWhitespaceControlStrictly = JinjavaInterpreter
-      .getCurrentMaybe()
-      .map(JinjavaInterpreter::getConfig)
-      .map(JinjavaConfig::getLegacyOverrides)
-      .map(LegacyOverrides::isParseWhitespaceControlStrictly)
-      .orElse(false);
-
-    WhitespaceControlParser parser = parseWhitespaceControlStrictly
-      ? WhitespaceControlParser.STRICT
-      : WhitespaceControlParser.LENIENT;
-
     String result = unwrapped;
-    if (parser.hasLeftTrim(result)) {
+    if (whitespaceControlParser.hasLeftTrim(result)) {
       setLeftTrim(true);
-      result = parser.stripLeft(result);
+      result = whitespaceControlParser.stripLeft(result);
     }
-    if (parser.hasRightTrim(result)) {
+    if (whitespaceControlParser.hasRightTrim(result)) {
       setRightTrim(true);
-      result = parser.stripRight(result);
+      result = whitespaceControlParser.stripRight(result);
     }
     return result;
   }
@@ -132,18 +121,43 @@ public String toString() {
   static Token newToken(
     int tokenKind,
     TokenScannerSymbols symbols,
+    WhitespaceControlParser whitespaceControlParser,
     String image,
     int lineNumber,
     int startPosition
   ) {
     if (tokenKind == symbols.getFixed()) {
-      return new TextToken(image, lineNumber, startPosition, symbols);
+      return new TextToken(
+        image,
+        lineNumber,
+        startPosition,
+        symbols,
+        whitespaceControlParser
+      );
     } else if (tokenKind == symbols.getNote()) {
-      return new NoteToken(image, lineNumber, startPosition, symbols);
+      return new NoteToken(
+        image,
+        lineNumber,
+        startPosition,
+        symbols,
+        whitespaceControlParser
+      );
     } else if (tokenKind == symbols.getExprStart()) {
-      return new ExpressionToken(image, lineNumber, startPosition, symbols);
+      return new ExpressionToken(
+        image,
+        lineNumber,
+        startPosition,
+        symbols,
+        whitespaceControlParser
+      );
     } else if (tokenKind == symbols.getTag()) {
-      return new TagToken(image, lineNumber, startPosition, symbols);
+      return new TagToken(
+        image,
+        lineNumber,
+        startPosition,
+        symbols,
+        whitespaceControlParser
+      );
     } else {
       throw new UnexpectedTokenException(
         String.valueOf((char) tokenKind),
diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java
@@ -19,6 +19,7 @@
 
 import com.google.common.collect.AbstractIterator;
 import com.hubspot.jinjava.JinjavaConfig;
+import com.hubspot.jinjava.features.BuiltInFeatures;
 
 public class TokenScanner extends AbstractIterator<Token> {
 
@@ -38,7 +39,8 @@ public class TokenScanner extends AbstractIterator<Token> {
   private char inQuote = 0;
   private int currLine = 1;
   private int lastNewlinePos = 0;
-  private TokenScannerSymbols symbols;
+  private final TokenScannerSymbols symbols;
+  private final WhitespaceControlParser whitespaceControlParser;
 
   public TokenScanner(String input, JinjavaConfig config) {
     this.config = config;
@@ -58,6 +60,10 @@ public TokenScanner(String input, JinjavaConfig config) {
     lastNewlinePos = 0;
 
     symbols = config.getTokenScannerSymbols();
+    whitespaceControlParser =
+      config.getLegacyOverrides().isParseWhitespaceControlStrictly()
+        ? WhitespaceControlParser.STRICT
+        : WhitespaceControlParser.LENIENT;
   }
 
   private Token getNextToken() {
@@ -88,7 +94,11 @@ private Token getNextToken() {
         if (currPost < length) {
           c = is[currPost];
           boolean startTokenFound = true;
-          if (config.getLegacyOverrides().isWhitespaceRequiredWithinTokens()) {
+          if (
+            config
+              .getFeatures()
+              .isActive(BuiltInFeatures.WHITESPACE_REQUIRED_WITHIN_TOKENS)
+          ) {
             boolean hasNextChar = (currPost + 1) < length;
             boolean nextCharIsWhitespace = hasNextChar && (' ' == is[currPost + 1]);
             startTokenFound = nextCharIsWhitespace;
@@ -232,12 +242,14 @@ private Token getEndToken() {
         String.valueOf(is, tokenStart, tokenLength),
         currLine,
         tokenStart - lastNewlinePos + 1,
-        symbols
+        symbols,
+        whitespaceControlParser
       );
     }
     return Token.newToken(
       type,
       symbols,
+      whitespaceControlParser,
       String.valueOf(is, tokenStart, tokenLength),
       currLine,
       tokenStart - lastNewlinePos + 1
@@ -248,6 +260,7 @@ private Token newToken(int kind) {
     Token t = Token.newToken(
       kind,
       symbols,
+      whitespaceControlParser,
       String.valueOf(is, lastStart, tokenLength),
       currLine,
       lastStart - lastNewlinePos + 1
@@ -276,7 +289,14 @@ private Token newToken(int kind) {
     }
 
     if (inRaw > 0 && t.getType() != symbols.getFixed()) {
-      return Token.newToken(symbols.getFixed(), symbols, t.image, currLine, tokenStart);
+      return Token.newToken(
+        symbols.getFixed(),
+        symbols,
+        whitespaceControlParser,
+        t.image,
+        currLine,
+        tokenStart
+      );
     }
 
     return t;
diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/UnclosedToken.java b/src/main/java/com/hubspot/jinjava/tree/parse/UnclosedToken.java
@@ -8,6 +8,16 @@ public UnclosedToken(
     int startPosition,
     TokenScannerSymbols symbols
   ) {
-    super(image, lineNumber, startPosition, symbols);
+    this(image, lineNumber, startPosition, symbols, WhitespaceControlParser.LENIENT);
+  }
+
+  public UnclosedToken(
+    String image,
+    int lineNumber,
+    int startPosition,
+    TokenScannerSymbols symbols,
+    WhitespaceControlParser whitespaceControlParser
+  ) {
+    super(image, lineNumber, startPosition, symbols, whitespaceControlParser);
   }
 }
diff --git a/src/test/java/com/hubspot/jinjava/tree/parse/TokenScannerTest.java b/src/test/java/com/hubspot/jinjava/tree/parse/TokenScannerTest.java
