feature #2996 Add "filter", "map", and "reduce" filters (fabpot)

This PR was squashed before being merged into the 1.x branch (closes #2996).

Discussion
----------

Add "filter", "map", and "reduce" filters

This PR adds support for 3 new filters: `filter`, `map`, and `reduce`. They take an arrow function as an argument (a PHP closure).

I have restricted the usage of arrow functions as much as possible as it makes no sense to support them everywhere. So, for now, they are only accepted as arguments to filters (using them as arguments to function is not supported but could be easily added if we have a use case).

The syntax is the following: `(x) => x + 3` where `(x)` is the list of arguments, `=>` starts the body, and the body is any Twig expression. Within the arrow function, the context is also available: `(x) => x + offset` works if `offset` is defined in the current context.

~~These new filters will allow us to deprecate the `if` support on the `for` tag, which does not work well with the `loop` variable:~~

```twig

{% set sizes = {xs: 34, s: 36, m: 38, l: 40, xl: 42} %}

{# before #}
{% for name, size in sizes if size < 38 %}
    {{ name }} = {{ size }}
    {% loop.last ? 'LAST' %} {# <--- works with this PR #}
{% endfor %}

{# after #}
{% for name, size in sizes|filter(size => size < 38) %}
    {{ name }} = {{ size }}
    {{ loop.last ? 'LAST' }}
{% endfor %}
```

This closes #2785

Commits
-------

5c15f89 added the key to the map and filter filters
13274bb added support for iterators
175041e removed fn in front of arrow functions
dc27763 changed arrow syntax
b30bce1 added "filter", "map", and "reduce" filters

Author: fabpot

CHANGELOG

@@ -1,5 +1,6 @@
 * 1.41.0 (2019-XX-XX)
 
+ * added "filter", "map", and "reduce" filters (and support for arrow functions)
  * fixed partial output leak when a PHP fatal error occurs
  * optimized context access on PHP 7.4
 

doc/filters/filter.rst

@@ -0,0 +1,47 @@
+``filter``
+=========
+
+.. versionadded:: 1.41
+    The ``filter`` filter was added in Twig 1.41.
+
+The ``filter`` filter filters elements of a sequence or a mapping using an arrow
+function. The arrow function receives the value of the sequence or mapping:
+
+.. code-block:: twig
+
+    {% set sizes = [34, 36, 38, 40, 42] %}
+
+    {% for v in sizes|filter(v => v > 38) -%}
+        {{ v }}
+    {% endfor %}
+    {# output 40 42 #}
+
+    {% set sizes = {
+        xs: 34,
+        s:  36,
+        m:  38,
+        l:  40,
+        xl: 42,
+    } %}
+
+    {% for k, v in sizes|filter(v => v > 38) -%}
+        {{ k }} = {{ v }}
+    {% endfor %}
+    {# output l = 40 xl = 42 #}
+
+The arrow function also receives the key as a second argument:
+
+.. code-block:: twig
+
+    {% for k, v in sizes|filter((v, k) => v > 38 and k != "xl") -%}
+        {{ k }} = {{ v }}
+    {% endfor %}
+    {# output l = 40 #}
+
+Note that the arrow function has access to the current context.
+
+Arguments
+---------
+
+* ``array``: The sequence or mapping
+* ``arrow``: The arrow function

doc/filters/index.rst

@@ -12,6 +12,7 @@ Filters
     date_modify
     default
     escape
+    filter
     first
     format
     join
@@ -20,10 +21,12 @@ Filters
     last
     length
     lower
+    map
     merge
     nl2br
     number_format
     raw
+    reduce
     replace
     reverse
     round

doc/filters/join.rst

@@ -19,15 +19,15 @@ define it with the optional first parameter:
 
     {{ [1, 2, 3]|join('|') }}
     {# outputs 1|2|3 #}
-    
+
 A second parameter can also be provided that will be the separator used between
 the last two items of the sequence:
 
 .. code-block:: twig
 
     {{ [1, 2, 3]|join(', ', ' and ') }}
     {# outputs 1, 2 and 3 #}
-    
+
 Arguments
 ---------
 

doc/filters/map.rst

@@ -0,0 +1,38 @@
+``map``
+=======
+
+.. versionadded:: 1.41
+    The ``map`` filter was added in Twig 1.41.
+
+The ``map`` filter applies an arrow function to the elements of a sequence or a
+mapping. The arrow function receives the value of the sequence or mapping:
+
+.. code-block:: twig
+
+    {% set people = [
+        {first: "Bob", last: "Smith"},
+        {first: "Alice", last: "Dupond"},
+    ] %}
+
+    {{ people|map(p => "#{p.first} #{p.last}")|join(', ') }}
+    {# outputs Bob Smith, Alice Dupond #}
+
+The arrow function also receives the key as a second argument:
+
+.. code-block:: twig
+
+    {% set people = {
+        "Bob": "Smith",
+        "Alice": "Dupond",
+    } %}
+
+    {{ people|map((first, last) => "#{first} #{last}")|join(', ') }}
+    {# outputs Bob Smith, Alice Dupond #}
+
+Note that the arrow function has access to the current context.
+
+Arguments
+---------
+
+* ``array``: The sequence or mapping
+* ``arrow``: The arrow function

doc/filters/reduce.rst

@@ -0,0 +1,33 @@
+``reduce``
+=========
+
+.. versionadded:: 1.41
+    The ``reduce`` filter was added in Twig 1.41.
+
+The ``reduce`` filter iteratively reduces a sequence or a mapping to a single
+value using an arrow function, so as to reduce it to a single value. The arrow
+function receives the return value of the previous iteration and the current
+value of the sequence or mapping:
+
+.. code-block:: twig
+
+    {% set numbers = [1, 2, 3] %}
+
+    {{ numbers|reduce((carry, v) => carry + v) }}
+    {# output 6 #}
+
+The ``reduce`` filter takes an ``initial`` value as a second argument:
+
+.. code-block:: twig
+
+    {{ numbers|reduce((carry, v) => carry + v, 10) }}
+    {# output 16 #}
+
+Note that the arrow function has access to the current context.
+
+Arguments
+---------
+
+* ``array``: The sequence or mapping
+* ``arrow``: The arrow function
+* ``initial``: The initial value

src/ExpressionParser.php

@@ -14,6 +14,7 @@
 
 use Twig\Error\SyntaxError;
 use Twig\Node\Expression\ArrayExpression;
+use Twig\Node\Expression\ArrowFunctionExpression;
 use Twig\Node\Expression\AssignNameExpression;
 use Twig\Node\Expression\Binary\ConcatBinary;
 use Twig\Node\Expression\BlockReferenceExpression;
@@ -68,8 +69,12 @@ public function __construct(Parser $parser, $env = null)
         }
     }
 
-    public function parseExpression($precedence = 0)
+    public function parseExpression($precedence = 0, $allowArrow = false)
     {
+        if ($allowArrow && $arrow = $this->parseArrow()) {
+            return $arrow;
+        }
+
         $expr = $this->getPrimary();
         $token = $this->parser->getCurrentToken();
         while ($this->isBinary($token) && $this->binaryOperators[$token->getValue()]['precedence'] >= $precedence) {
@@ -98,6 +103,64 @@ public function parseExpression($precedence = 0)
         return $expr;
     }
 
+    /**
+     * @return ArrowFunctionExpression|null
+     */
+    private function parseArrow()
+    {
+        $stream = $this->parser->getStream();
+
+        // short array syntax (one argument, no parentheses)?
+        if ($stream->look(1)->test(Token::ARROW_TYPE)) {
+            $line = $stream->getCurrent()->getLine();
+            $token = $stream->expect(Token::NAME_TYPE);
+            $names = [new AssignNameExpression($token->getValue(), $token->getLine())];
+            $stream->expect(Token::ARROW_TYPE);
+
+            return new ArrowFunctionExpression($this->parseExpression(0), new Node($names), $line);
+        }
+
+        // first, determine if we are parsing an arrow function by finding => (long form)
+        $i = 0;
+        if (!$stream->look($i)->test(Token::PUNCTUATION_TYPE, '(')) {
+            return null;
+        }
+        ++$i;
+        while (true) {
+            // variable name
+            ++$i;
+            if (!$stream->look($i)->test(Token::PUNCTUATION_TYPE, ',')) {
+                break;
+            }
+            ++$i;
+        }
+        if (!$stream->look($i)->test(Token::PUNCTUATION_TYPE, ')')) {
+            return null;
+        }
+        ++$i;
+        if (!$stream->look($i)->test(Token::ARROW_TYPE)) {
+            return null;
+        }
+
+        // yes, let's parse it properly
+        $token = $stream->expect(Token::PUNCTUATION_TYPE, '(');
+        $line = $token->getLine();
+
+        $names = [];
+        while (true) {
+            $token = $stream->expect(Token::NAME_TYPE);
+            $names[] = new AssignNameExpression($token->getValue(), $token->getLine());
+
+            if (!$stream->nextIf(Token::PUNCTUATION_TYPE, ',')) {
+                break;
+            }
+        }
+        $stream->expect(Token::PUNCTUATION_TYPE, ')');
+        $stream->expect(Token::ARROW_TYPE);
+
+        return new ArrowFunctionExpression($this->parseExpression(0), new Node($names), $line);
+    }
+
     protected function getPrimary()
     {
         $token = $this->parser->getCurrentToken();
@@ -499,7 +562,7 @@ public function parseFilterExpressionRaw($node, $tag = null)
             if (!$this->parser->getStream()->test(Token::PUNCTUATION_TYPE, '(')) {
                 $arguments = new Node();
             } else {
-                $arguments = $this->parseArguments(true);
+                $arguments = $this->parseArguments(true, false, true);
             }
 
             $class = $this->getFilterNodeClass($name->getAttribute('value'), $token->getLine());
@@ -526,7 +589,7 @@ public function parseFilterExpressionRaw($node, $tag = null)
      *
      * @throws SyntaxError
      */
-    public function parseArguments($namedArguments = false, $definition = false)
+    public function parseArguments($namedArguments = false, $definition = false, $allowArrow = false)
     {
         $args = [];
         $stream = $this->parser->getStream();
@@ -541,7 +604,7 @@ public function parseArguments($namedArguments = false, $definition = false)
                 $token = $stream->expect(Token::NAME_TYPE, null, 'An argument must be a name');
                 $value = new NameExpression($token->getValue(), $this->parser->getCurrentToken()->getLine());
             } else {
-                $value = $this->parseExpression();
+                $value = $this->parseExpression(0, $allowArrow);
             }
 
             $name = null;
@@ -558,7 +621,7 @@ public function parseArguments($namedArguments = false, $definition = false)
                         throw new SyntaxError(sprintf('A default value for an argument must be a constant (a boolean, a string, a number, or an array).'), $token->getLine(), $stream->getSourceContext());
                     }
                 } else {
-                    $value = $this->parseExpression();
+                    $value = $this->parseExpression(0, $allowArrow);
                 }
             }
 

src/Extension/CoreExtension.php

@@ -194,6 +194,9 @@ public function getFilters()
             new TwigFilter('sort', 'twig_sort_filter'),
             new TwigFilter('merge', 'twig_array_merge'),
             new TwigFilter('batch', 'twig_array_batch'),
+            new TwigFilter('filter', 'twig_array_filter'),
+            new TwigFilter('map', 'twig_array_map'),
+            new TwigFilter('reduce', 'twig_array_reduce'),
 
             // string/array filters
             new TwigFilter('reverse', 'twig_reverse_filter', ['needs_environment' => true]),
@@ -1682,4 +1685,36 @@ function twig_array_batch($items, $size, $fill = null, $preserveKeys = true)
 
     return $result;
 }
+
+function twig_array_filter($array, $arrow)
+{
+    if (\is_array($array)) {
+        if (\PHP_VERSION_ID >= 50600) {
+            return array_filter($array, $arrow, \ARRAY_FILTER_USE_BOTH);
+        }
+
+        return array_filter($array, $arrow);
+    }
+
+    return new \CallbackFilterIterator($array, $arrow);
+}
+
+function twig_array_map($array, $arrow)
+{
+    $r = [];
+    foreach ($array as $k => $v) {
+        $r[$k] = $arrow($v, $k);
+    }
+
+    return $r;
+}
+
+function twig_array_reduce($array, $arrow, $initial = null)
+{
+    if (!\is_array($array)) {
+        $array = iterator_to_array($array);
+    }
+
+    return array_reduce($array, $arrow, $initial);
+}
 }

src/Lexer.php

@@ -333,8 +333,13 @@ protected function lexExpression()
             }
         }
 
+        // arrow function
+        if ('=' === $this->code[$this->cursor] && '>' === $this->code[$this->cursor + 1]) {
+            $this->pushToken(Token::ARROW_TYPE, '=>');
+            $this->moveCursor('=>');
+        }
         // operators
-        if (preg_match($this->regexes['operator'], $this->code, $match, 0, $this->cursor)) {
+        elseif (preg_match($this->regexes['operator'], $this->code, $match, 0, $this->cursor)) {
             $this->pushToken(Token::OPERATOR_TYPE, preg_replace('/\s+/', ' ', $match[0]));
             $this->moveCursor($match[0]);
         }