refactor: Reuse FilteredValueToken and fix architectural layering

src/context/context.ts

@@ -31,14 +31,18 @@ export class Context {
    * The normalized liquid options object
    */
   public opts: NormalizedFullOptions
+  /**
+   * Reference to the Liquid instance for filter resolution
+   */
+  public liquid?: any
   /**
    * Throw when accessing undefined variable?
    */
   public strictVariables: boolean;
   public ownPropertyOnly: boolean;
   public memoryLimit: Limiter;
   public renderLimit: Limiter;
-  public constructor (env: object = {}, opts: NormalizedFullOptions = defaultOptions, renderOptions: RenderOptions = {}, { memoryLimit, renderLimit }: { [key: string]: Limiter } = {}) {
+  public constructor (env: object = {}, opts: NormalizedFullOptions = defaultOptions, renderOptions: RenderOptions = {}, { memoryLimit, renderLimit, liquid }: { memoryLimit?: Limiter, renderLimit?: Limiter, liquid?: any } = {}) {
     this.sync = !!renderOptions.sync
     this.opts = opts
     this.globals = renderOptions.globals ?? opts.globals
@@ -47,6 +51,7 @@ export class Context {
     this.ownPropertyOnly = renderOptions.ownPropertyOnly ?? opts.ownPropertyOnly
     this.memoryLimit = memoryLimit ?? new Limiter('memory alloc', renderOptions.memoryLimit ?? opts.memoryLimit)
     this.renderLimit = renderLimit ?? new Limiter('template render', getPerformance().now() + (renderOptions.renderLimit ?? opts.renderLimit))
+    this.liquid = liquid
   }
   public getRegister (key: string) {
     return (this.registers[key] = this.registers[key] || {})
@@ -109,7 +114,8 @@ export class Context {
       strictVariables: this.strictVariables
     }, {
       renderLimit: this.renderLimit,
-      memoryLimit: this.memoryLimit
+      memoryLimit: this.memoryLimit,
+      liquid: this.liquid
     })
   }
   private findScope (key: string | number) {

src/index.ts

@@ -11,7 +11,7 @@ export { Context, Scope } from './context'
 export { Value, Hash, Template, FilterImplOptions, Tag, Filter, Output, Variable, VariableLocation, VariableSegments, Variables, StaticAnalysis, StaticAnalysisOptions, analyze, analyzeSync, Arguments, PartialScope } from './template'
 export type { TagRenderReturn } from './template'
 export { Token, TopLevelToken, TagToken, ValueToken } from './tokens'
-export type { RangeToken, LiteralToken, QuotedToken, PropertyAccessToken, NumberToken } from './tokens'
+export type { RangeToken, LiteralToken, QuotedToken, PropertyAccessToken, NumberToken, FilteredValueToken } from './tokens'
 export { TokenKind, Tokenizer, ParseStream, Parser } from './parser'
 export { filters } from './filters'
 export * from './tags'

src/liquid-options.ts

@@ -84,6 +84,8 @@ export interface LiquidOptions {
   operators?: Operators;
   /** Respect parameter order when using filters like "for ... reversed limit", Defaults to `false`. */
   orderedFilterParameters?: boolean;
+  /** Allow parenthesized expressions as operands in conditions and loops, e.g. `{% if (foo | upcase) == "BAR" %}`. This is a non-standard extension to Liquid. Defaults to `false`. */
+  groupedExpressions?: boolean;
   /** For DoS handling, limit total length of templates parsed in one `parse()` call. A typical PC can handle 1e8 (100M) characters without issues. */
   parseLimit?: number;
   /** For DoS handling, limit total time (in ms) for each `render()` call. */
@@ -159,6 +161,7 @@ export interface NormalizedFullOptions extends NormalizedOptions {
   globals: object;
   keepOutputType: boolean;
   operators: Operators;
+  groupedExpressions: boolean;
   parseLimit: number;
   renderLimit: number;
   memoryLimit: number;
@@ -195,6 +198,7 @@ export const defaultOptions: NormalizedFullOptions = {
   globals: {},
   keepOutputType: false,
   operators: defaultOperators,
+  groupedExpressions: false,
   memoryLimit: Infinity,
   parseLimit: Infinity,
   renderLimit: Infinity

src/liquid.ts

@@ -31,7 +31,7 @@ export class Liquid {
   }
 
   public _render (tpl: Template[], scope: Context | object | undefined, renderOptions: RenderOptions): IterableIterator<any> {
-    const ctx = scope instanceof Context ? scope : new Context(scope, this.options, renderOptions)
+    const ctx = scope instanceof Context ? scope : new Context(scope, this.options, renderOptions, { liquid: this })
     return this.renderer.renderTemplates(tpl, ctx)
   }
   public async render (tpl: Template[], scope?: object, renderOptions?: RenderOptions): Promise<any> {
@@ -41,7 +41,7 @@ export class Liquid {
     return toValueSync(this._render(tpl, scope, { ...renderOptions, sync: true }))
   }
   public renderToNodeStream (tpl: Template[], scope?: object, renderOptions: RenderOptions = {}): NodeJS.ReadableStream {
-    const ctx = new Context(scope, this.options, renderOptions)
+    const ctx = new Context(scope, this.options, renderOptions, { liquid: this })
     return this.renderer.renderTemplatesToNodeStream(tpl, ctx)
   }
 
@@ -88,7 +88,7 @@ export class Liquid {
 
   public _evalValue (str: string, scope?: object | Context): IterableIterator<any> {
     const value = new Value(str, this)
-    const ctx = scope instanceof Context ? scope : new Context(scope, this.options)
+    const ctx = scope instanceof Context ? scope : new Context(scope, this.options, {}, { liquid: this })
     return value.value(ctx)
   }
   public async evalValue (str: string, scope?: object | Context): Promise<any> {

src/parser/parser.ts

@@ -33,7 +33,7 @@ export class Parser {
   public parse (html: string, filepath?: string): Template[] {
     html = String(html)
     this.parseLimit.use(html.length)
-    const tokenizer = new Tokenizer(html, this.liquid.options.operators, filepath)
+    const tokenizer = new Tokenizer(html, this.liquid.options.operators, filepath, undefined, this.liquid.options.groupedExpressions)
     const tokens = tokenizer.readTopLevelTokens(this.liquid.options)
     return this.parseTokens(tokens)
   }

src/parser/token-kind.ts

@@ -12,5 +12,6 @@ export enum TokenKind {
   Quoted = 1024,
   Operator = 2048,
   FilteredValue = 4096,
+  GroupedExpression = 8192,
   Delimited = Tag | Output
 }

src/parser/tokenizer.spec.ts

@@ -1,4 +1,4 @@
-import { LiquidTagToken, HTMLToken, QuotedToken, OutputToken, TagToken, OperatorToken, RangeToken, PropertyAccessToken, NumberToken, IdentifierToken } from '../tokens'
+import { LiquidTagToken, HTMLToken, QuotedToken, OutputToken, TagToken, OperatorToken, RangeToken, PropertyAccessToken, NumberToken, IdentifierToken, FilteredValueToken } from '../tokens'
 import { Tokenizer } from './tokenizer'
 import { defaultOperators } from '../render/operator'
 import { createTrie } from '../util/operator-trie'
@@ -229,24 +229,76 @@ describe('Tokenizer', function () {
   })
   describe('#readRange()', () => {
     it('should read `(1..3)`', () => {
-      const range = new Tokenizer('(1..3)').readRange()
+      const range = new Tokenizer('(1..3)').readGroupOrRange()
+      expect(range).toBeDefined()
       expect(range).toBeInstanceOf(RangeToken)
       expect(range!.getText()).toEqual('(1..3)')
-      const { lhs, rhs } = range!
-      expect(lhs).toBeInstanceOf(NumberToken)
-      expect(lhs.getText()).toBe('1')
-      expect(rhs).toBeInstanceOf(NumberToken)
-      expect(rhs.getText()).toBe('3')
+      expect((range as RangeToken).lhs).toBeInstanceOf(NumberToken)
+      expect((range as RangeToken).lhs.getText()).toBe('1')
+      expect((range as RangeToken).rhs).toBeInstanceOf(NumberToken)
+      expect((range as RangeToken).rhs.getText()).toBe('3')
     })
     it('should throw for `(..3)`', () => {
-      expect(() => new Tokenizer('(..3)').readRange()).toThrow('unexpected token "..3)", value expected')
+      expect(() => new Tokenizer('(..3)').readGroupOrRange()).toThrow('unexpected token "..3)", value expected')
     })
     it('should read `(a.b..c["..d"])`', () => {
-      const range = new Tokenizer('(a.b..c["..d"])').readRange()
+      const range = new Tokenizer('(a.b..c["..d"])').readGroupOrRange()
+      expect(range).toBeDefined()
       expect(range).toBeInstanceOf(RangeToken)
       expect(range!.getText()).toEqual('(a.b..c["..d"])')
     })
   })
+  describe('#readGroupedExpression()', () => {
+    function createGrouped (input: string): Tokenizer {
+      const t = new Tokenizer(input, defaultOperators)
+      t.groupedExpressions = true
+      return t
+    }
+    it('should read `(foo | upcase)` as FilteredValueToken', () => {
+      const token = createGrouped('(foo | upcase)').readValue()
+      expect(token).toBeInstanceOf(FilteredValueToken)
+      const grouped = token as FilteredValueToken
+      expect(grouped.getText()).toBe('(foo | upcase)')
+      expect(grouped.initial.postfix).toHaveLength(1)
+      expect(grouped.filters).toHaveLength(1)
+      expect(grouped.filters[0].name).toBe('upcase')
+    })
+    it('should read `(foo | append: "!")` with filter argument', () => {
+      const token = createGrouped('(foo | append: "!")').readValue()
+      expect(token).toBeInstanceOf(FilteredValueToken)
+      const grouped = token as FilteredValueToken
+      expect(grouped.filters).toHaveLength(1)
+      expect(grouped.filters[0].name).toBe('append')
+      expect(grouped.filters[0].args).toHaveLength(1)
+    })
+    it('should read nested `((foo | append: "!") | upcase)`', () => {
+      const token = createGrouped('((foo | append: "!") | upcase)').readValue()
+      expect(token).toBeInstanceOf(FilteredValueToken)
+      const grouped = token as FilteredValueToken
+      expect(grouped.filters).toHaveLength(1)
+      expect(grouped.filters[0].name).toBe('upcase')
+      expect(grouped.initial.postfix).toHaveLength(1)
+      expect(grouped.initial.postfix[0]).toBeInstanceOf(FilteredValueToken)
+    })
+    it('should parse `(a | upcase) == "BAR"` as expression', () => {
+      const exp = [...createGrouped('(a | upcase) == "BAR"').readExpressionTokens()]
+      expect(exp).toHaveLength(3)
+      expect(exp[0]).toBeInstanceOf(FilteredValueToken)
+      expect(exp[1]).toBeInstanceOf(OperatorToken)
+      expect(exp[1].getText()).toBe('==')
+      expect(exp[2]).toBeInstanceOf(QuotedToken)
+    })
+    it('should still parse `(1..3)` as RangeToken', () => {
+      const token = createGrouped('(1..3)').readValue()
+      expect(token).toBeInstanceOf(RangeToken)
+    })
+    it('should throw for unclosed parens', () => {
+      expect(() => createGrouped('(foo | upcase').readValue()).toThrow('unbalanced parentheses')
+    })
+    it('should fall back to readRange when flag is off', () => {
+      expect(() => new Tokenizer('(foo | upcase)', defaultOperators).readValue()).toThrow('invalid range syntax')
+    })
+  })
   describe('#readFilter()', () => {
     it('should read a simple filter', function () {
       const tokenizer = new Tokenizer('| plus')

src/parser/tokenizer.ts

@@ -9,6 +9,7 @@ import { whiteSpaceCtrl } from './whitespace-ctrl'
 export class Tokenizer {
   p: number
   N: number
+  public groupedExpressions: boolean
   private rawBeginAt = -1
   private opTrie: Trie<OperatorHandler>
   private literalTrie: Trie<LiteralValue>
@@ -17,12 +18,14 @@ export class Tokenizer {
     public input: string,
     operators: Operators = defaultOptions.operators,
     public file?: string,
-    range?: [number, number]
+    range?: [number, number],
+    groupedExpressions = false
   ) {
     this.p = range ? range[0] : 0
     this.N = range ? range[1] : input.length
     this.opTrie = createTrie(operators)
     this.literalTrie = createTrie(literalValues)
+    this.groupedExpressions = groupedExpressions
   }
 
   readExpression () {
@@ -80,6 +83,7 @@ export class Tokenizer {
   readFilter (): FilterToken | null {
     this.skipBlank()
     if (this.end()) return null
+    if (this.peek() === ')') return null
     this.assert(this.read() === '|', `expected "|" before filter`)
     const name = this.readIdentifier()
     if (!name.size()) {
@@ -94,9 +98,9 @@ export class Tokenizer {
         const arg = this.readFilterArg()
         arg && args.push(arg)
         this.skipBlank()
-        this.assert(this.end() || this.peek() === ',' || this.peek() === '|', () => `unexpected character ${this.snapshot()}`)
+        this.assert(this.end() || this.peek() === ',' || this.peek() === '|' || this.peek() === ')', () => `unexpected character ${this.snapshot()}`)
       } while (this.peek() === ',')
-    } else if (this.peek() === '|' || this.end()) {
+    } else if (this.peek() === '|' || this.peek() === ')' || this.end()) {
       // do nothing
     } else {
       throw this.error('expected ":" after filter name')
@@ -307,10 +311,14 @@ export class Tokenizer {
     return -1
   }
 
-  readValue (): ValueToken | undefined {
+  readValue (): ValueToken | FilteredValueToken | undefined {
     this.skipBlank()
     const begin = this.p
-    const variable = this.readLiteral() || this.readQuoted() || this.readRange() || this.readNumber()
+    let variable: ValueToken | FilteredValueToken | undefined = this.readLiteral() || this.readQuoted()
+    if (!variable && this.peek() === '(') {
+      variable = this.readGroupOrRange()
+    }
+    variable = variable || this.readNumber()
     const props = this.readProperties(!variable)
     if (!props.length) return variable
     return new PropertyAccessToken(variable, props, this.input, begin, this.p)
@@ -385,18 +393,31 @@ export class Tokenizer {
     return literal
   }
 
-  readRange (): RangeToken | undefined {
+  readGroupOrRange (): FilteredValueToken | RangeToken | undefined {
     this.skipBlank()
     const begin = this.p
     if (this.peek() !== '(') return
     ++this.p
     const lhs = this.readValueOrThrow()
     this.skipBlank()
-    this.assert(this.read() === '.' && this.read() === '.', 'invalid range syntax')
-    const rhs = this.readValueOrThrow()
-    this.skipBlank()
-    this.assert(this.read() === ')', 'invalid range syntax')
-    return new RangeToken(this.input, begin, this.p, lhs, rhs, this.file)
+
+    if (this.peek() === '.' && this.peek(1) === '.') {
+      this.p += 2
+      const rhs = this.readValueOrThrow()
+      this.skipBlank()
+      this.assert(this.read() === ')', 'invalid range syntax')
+      return new RangeToken(this.input, begin, this.p, lhs, rhs, this.file)
+    }
+
+    if (this.groupedExpressions) {
+      const expression = new Expression((function * () { yield lhs })())
+      const filters = this.readFilters()
+      this.skipBlank()
+      this.assert(this.read() === ')', 'unbalanced parentheses')
+      return new FilteredValueToken(expression, filters, this.input, begin, this.p, this.file)
+    }
+
+    throw this.error('invalid range syntax')
   }
 
   readValueOrThrow (): ValueToken {

src/render/expression.ts

@@ -1,8 +1,9 @@
-import { QuotedToken, RangeToken, OperatorToken, Token, PropertyAccessToken, OperatorType, operatorTypes } from '../tokens'
-import { isRangeToken, isPropertyAccessToken, UndefinedVariableError, range, isOperatorToken, assert } from '../util'
+import { QuotedToken, RangeToken, OperatorToken, Token, PropertyAccessToken, OperatorType, operatorTypes, FilteredValueToken } from '../tokens'
+import { isRangeToken, isPropertyAccessToken, isFilteredValueToken, UndefinedVariableError, range, isOperatorToken, assert } from '../util'
 import type { Context } from '../context'
 import type { UnaryOperatorHandler } from '../render'
 import { Drop } from '../drop'
+import { Filter } from '../template/filter'
 
 export class Expression {
   readonly postfix: Token[]
@@ -40,6 +41,22 @@ export function * evalToken (token: Token | undefined, ctx: Context, lenient = f
   if ('content' in token) return token.content
   if (isPropertyAccessToken(token)) return yield evalPropertyAccessToken(token, ctx, lenient)
   if (isRangeToken(token)) return yield evalRangeToken(token, ctx)
+  if (isFilteredValueToken(token)) return yield evalFilteredValueToken(token, ctx, lenient)
+}
+
+function * evalFilteredValueToken (token: FilteredValueToken, ctx: Context, lenient: boolean): IterableIterator<unknown> {
+  assert(ctx.liquid, 'FilteredValueToken evaluation requires liquid instance in context')
+  lenient = lenient || (ctx.opts.lenientIf && token.filters.length > 0 && token.filters[0].name === 'default')
+  let val = yield token.initial.evaluate(ctx, lenient)
+
+  for (const filterToken of token.filters) {
+    const filterImpl = ctx.liquid.filters[filterToken.name]
+    assert(filterImpl || !ctx.liquid.options.strictFilters, () => `undefined filter: ${filterToken.name}`)
+    const filter = new Filter(filterToken, filterImpl, ctx.liquid)
+    val = yield filter.render(val, ctx)
+  }
+
+  return val
 }
 
 function * evalPropertyAccessToken (token: PropertyAccessToken, ctx: Context, lenient: boolean): IterableIterator<unknown> {

src/tags/case.ts

@@ -1,11 +1,11 @@
-import { ValueToken, Liquid, toValue, evalToken, Value, Emitter, TagToken, TopLevelToken, Context, Template, Tag, ParseStream } from '..'
+import { ValueToken, Liquid, toValue, evalToken, Value, Emitter, TagToken, TopLevelToken, Context, Template, Tag, ParseStream, FilteredValueToken } from '..'
 import { Parser } from '../parser'
 import { equals } from '../render'
 import { Arguments } from '../template'
 
 export default class extends Tag {
   value: Value
-  branches: { values: ValueToken[], templates: Template[] }[] = []
+  branches: { values: (ValueToken | FilteredValueToken)[], templates: Template[] }[] = []
   elseTemplates: Template[] = []
   constructor (tagToken: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) {
     super(tagToken, remainTokens, liquid)
@@ -22,7 +22,7 @@ export default class extends Tag {
 
         p = []
 
-        const values: ValueToken[] = []
+        const values: (ValueToken | FilteredValueToken)[] = []
         while (!token.tokenizer.end()) {
           values.push(token.tokenizer.readValueOrThrow())
           token.tokenizer.skipBlank()

src/tags/for.ts

@@ -1,4 +1,4 @@
-import { Hash, ValueToken, Liquid, Tag, evalToken, Emitter, TagToken, TopLevelToken, Context, Template, ParseStream } from '..'
+import { Hash, ValueToken, Liquid, Tag, evalToken, Emitter, TagToken, TopLevelToken, Context, Template, ParseStream, FilteredValueToken } from '..'
 import { assertEmpty, isValueToken, toEnumerable } from '../util'
 import { ForloopDrop } from '../drop/forloop-drop'
 import { Parser } from '../parser'
@@ -10,7 +10,7 @@ type valueOf<T> = T[keyof T]
 
 export default class extends Tag {
   variable: string
-  collection: ValueToken
+  collection: ValueToken | FilteredValueToken
   hash: Hash
   templates: Template[]
   elseTemplates: Template[]

src/tags/tablerow.ts

@@ -1,5 +1,5 @@
 import { isValueToken, toEnumerable } from '../util'
-import { ValueToken, Liquid, Tag, evalToken, Emitter, Hash, TagToken, TopLevelToken, Context, Template, ParseStream } from '..'
+import { ValueToken, Liquid, Tag, evalToken, Emitter, Hash, TagToken, TopLevelToken, Context, Template, ParseStream, FilteredValueToken } from '..'
 import { TablerowloopDrop } from '../drop/tablerowloop-drop'
 import { Parser } from '../parser'
 import { Arguments } from '../template'
@@ -8,7 +8,7 @@ export default class extends Tag {
   variable: string
   args: Hash
   templates: Template[]
-  collection: ValueToken
+  collection: ValueToken | FilteredValueToken
   constructor (tagToken: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) {
     super(tagToken, remainTokens, liquid)
     const variable = this.tokenizer.readIdentifier()