feat(data-explore): Add support in EAP to filter Array by elements (#7898)

Current implementation of arrays in TraceItemTable endpoint transforms
arrays as array of strings, and then only allows `LIKE/NOT_LIKE` filter
on them.

With this change, arrays will preserve their original type, and client
can filter EAP items by element of an array attribute.
To preserve the type, array is converted to JSON string,
(clickhouse-driver doesn't support Array(JSON) type, as discussed in
this [PR](#7551)), and post
processing of results recovers the original array by decoding JSON
string.

To filter, predicate maps arrays to corresponding type and use array
exists with element filter.

Example Clickhouse query is:
```

SELECT (
    cast(lower(leftPad(hex(item_id), if(
        greater(length(hex(item_id)), 16), 32, 16), '0')), 'String'
    ) AS `sentry.item_id_TYPE_STRING`
), (toJSONString(attributes_array.`tags`::Array(
    JSON)) AS tags_TYPE_ARRAY
)
FROM eap_items_1_local
WHERE in(
    project_id, [1, 2, 3]
)
AND equals(organization_id, 1)
AND less(
    timestamp, toDateTime('2026-04-24 14:00:00')
)
AND greaterOrEquals(timestamp, toDateTime('2026-04-24 08:00:00'))
AND arrayExists(x -> equals(lower(x), lower('error')), (arrayMap(x -> coalesce(
    x.`String`::Nullable(String), toString(x.`Int`::Nullable(Int64)), toString(x.`Double`::Nullable(Float64)), x.`Bool`::Nullable(String)), attributes_array.`tags`::Array(
        JSON)) AS tags_TYPE_ARRAY__array_members
    )
)
AND lessOrEquals(sampling_factor, 1)
AND greater(sampling_factor, 0)
AND equals(item_type, 1)
LIMIT 10001 OFFSET 0

```

Author: manessaraj

snuba/protos/common.py

@@ -120,6 +120,61 @@ def _generate_subscriptable_reference(
     )
 
 
+def type_array_to_membership_array_expression(attr_key: AttributeKey) -> FunctionCall:
+    """To be used only in WHERE clause, not SELECT"""
+    if attr_key.type != AttributeKey.Type.TYPE_ARRAY:
+        raise MalformedAttributeException(
+            f"type_array_to_membership_array_expression expected TYPE_ARRAY, got "
+            f"{AttributeKey.Type.Name(attr_key.type)}"
+        )
+    # We need different label than attribute_key_to_expression(TYPE_ARRAY) [toJSONString]
+    alias = f"{_build_label_mapping_key(attr_key)}__array_members"
+    x = Argument(None, "x")
+    return FunctionCall(
+        alias=alias,
+        function_name="arrayMap",
+        parameters=(
+            Lambda(
+                alias=None,
+                parameters=("x",),
+                transformation=FunctionCall(
+                    alias=None,
+                    function_name="coalesce",
+                    parameters=(
+                        JsonPath(None, x, "String", "Nullable(String)"),
+                        FunctionCall(
+                            None,
+                            "toString",
+                            (JsonPath(None, x, "Int", "Nullable(Int64)"),),
+                        ),
+                        FunctionCall(
+                            None,
+                            "toString",
+                            (JsonPath(None, x, "Double", "Nullable(Float64)"),),
+                        ),
+                        JsonPath(None, x, "Bool", "Nullable(String)"),
+                    ),
+                ),
+            ),
+            JsonPath(
+                alias=None,
+                base=column("attributes_array"),
+                path=attr_key.name,
+                return_type="Array(JSON)",
+            ),
+        ),
+    )
+
+
+def type_array_to_stored_array_json_path(attr_key: AttributeKey) -> JsonPath:
+    return JsonPath(
+        alias=None,
+        base=column("attributes_array"),
+        path=attr_key.name,
+        return_type="Array(JSON)",
+    )
+
+
 def attribute_key_to_expression(attr_key: AttributeKey) -> Expression:
     """Convert an AttributeKey proto to a Snuba Expression.
 
@@ -178,45 +233,13 @@ def attribute_key_to_expression(attr_key: AttributeKey) -> Expression:
             )
 
     if attr_key.type == AttributeKey.Type.TYPE_ARRAY:
-        # Array values are stored as tagged variants (e.g. {"String": "alice"},
-        # {"Int": "123"}) in the JSON column. Cast to Array(JSON), then extract
-        # the value from whichever variant tag is present. We coalesce across
-        # all supported types, converting non-string types via toString so the
-        # result is always a string array.
-        x = Argument(None, "x")
+        # Tagged array under attributes_array.* as Array(JSON). Select toJSONString(...)
+        # so the result column is String; callers decode in application code. Raw
+        # Array(JSON) is not returned in the SELECT to avoid native client limits.
         return FunctionCall(
             alias=alias,
-            function_name="arrayMap",
-            parameters=(
-                Lambda(
-                    alias=None,
-                    parameters=("x",),
-                    transformation=FunctionCall(
-                        alias=None,
-                        function_name="coalesce",
-                        parameters=(
-                            JsonPath(None, x, "String", "Nullable(String)"),
-                            FunctionCall(
-                                None,
-                                "toString",
-                                (JsonPath(None, x, "Int", "Nullable(Int64)"),),
-                            ),
-                            FunctionCall(
-                                None,
-                                "toString",
-                                (JsonPath(None, x, "Double", "Nullable(Float64)"),),
-                            ),
-                            JsonPath(None, x, "Bool", "Nullable(String)"),
-                        ),
-                    ),
-                ),
-                JsonPath(
-                    alias=None,
-                    base=column("attributes_array"),
-                    path=attr_key.name,
-                    return_type="Array(JSON)",
-                ),
-            ),
+            function_name="toJSONString",
+            parameters=(type_array_to_stored_array_json_path(attr_key),),
         )
 
     raise MalformedAttributeException(

snuba/web/rpc/common/common.py

@@ -12,7 +12,10 @@
 )
 
 from snuba import settings, state
-from snuba.protos.common import MalformedAttributeException
+from snuba.protos.common import (
+    MalformedAttributeException,
+    type_array_to_membership_array_expression,
+)
 from snuba.protos.common import (
     attribute_key_to_expression as _attribute_key_to_expression,
 )
@@ -54,6 +57,21 @@ def attribute_key_to_expression(attr_key: AttributeKey) -> Expression:
         raise BadSnubaRPCRequestException(str(e)) from e
 
 
+def _trace_item_filter_key_expression(
+    attr_to_key_expression_callable: Callable[[AttributeKey], Expression], key: AttributeKey
+) -> Expression:
+    """predicates must use the normalized
+    ``arrayMap`` (``type_array_to_membership_array_expression``) so
+    ``arrayExists`` compares per element. It is different from SELECT predicate.
+    """
+    if key.type == AttributeKey.Type.TYPE_ARRAY:
+        try:
+            return type_array_to_membership_array_expression(key)
+        except MalformedAttributeException as e:
+            raise BadSnubaRPCRequestException(str(e)) from e
+    return attr_to_key_expression_callable(key)
+
+
 Tin = TypeVar("Tin", bound=ProtobufMessage)
 Tout = TypeVar("Tout", bound=ProtobufMessage)
 
@@ -265,6 +283,76 @@ def _attribute_value_to_expression(v: AttributeValue) -> Expression:
 }
 
 
+def _validate_comparison_filter_type_array(
+    op: ComparisonFilter.Op.ValueType, v: AttributeValue
+) -> None:
+    if op in (ComparisonFilter.OP_LIKE, ComparisonFilter.OP_NOT_LIKE):
+        if v.WhichOneof("value") != "val_str":
+            raise BadSnubaRPCRequestException(
+                "LIKE/NOT_LIKE on array keys requires a string pattern"
+            )
+        return
+    if op in (ComparisonFilter.OP_EQUALS, ComparisonFilter.OP_NOT_EQUALS):
+        # Array can be empty or non-empty. It can never be null, or can never have null elements.
+        vt = v.WhichOneof("value")
+        if vt in (
+            None,
+            "val_null",
+            "val_array",
+            "val_str_array",
+            "val_int_array",
+            "val_float_array",
+            "val_double_array",
+        ):
+            raise BadSnubaRPCRequestException(
+                "OP_EQUALS/OP_NOT_EQUALS on array keys require a scalar value "
+                "(e.g. val_str, val_int) or null (is_null / val_null) to match null elements"
+            )
+        return
+    raise BadSnubaRPCRequestException(
+        f"{ComparisonFilter.Op.Name(op)} is not supported on array keys "
+        "(supported: LIKE, NOT_LIKE, OP_EQUALS, OP_NOT_EQUALS)"
+    )
+
+
+def _type_array_membership_rhs_expression(v: AttributeValue) -> Expression:
+    """RHS as String, comparable to TYPE_ARRAY arrayMap output (Array(String) in CH)."""
+    value_type = v.WhichOneof("value")
+    match value_type:
+        case "val_str":
+            return literal(v.val_str)
+        case "val_int":
+            return f.toString(literal(v.val_int))
+        case "val_double":
+            return f.toString(literal(v.val_double))
+        case "val_float":
+            return f.toString(literal(v.val_float))
+        case "val_bool":
+            return literal(str(v.val_bool).lower())
+        case _:
+            raise BadSnubaRPCRequestException(
+                f"unsupported AttributeValue for array membership: {value_type}"
+            )
+
+
+def _type_array_includes_scalar_expression(
+    array_expr: Expression,
+    v: AttributeValue,
+    ignore_case: bool,
+) -> Expression:
+    """Any element equals scalar (includes / [*])"""
+    if v.WhichOneof("value") == "val_null" or v.is_null:
+        raise BadSnubaRPCRequestException("Arrays can't be NULL or cannot have NULL elements")
+    x = Argument(None, "x")
+    rhs = _type_array_membership_rhs_expression(v)
+    if ignore_case and v.WhichOneof("value") == "val_str":
+        return f.arrayExists(
+            Lambda(None, ("x",), f.equals(f.lower(x), f.lower(rhs))),
+            array_expr,
+        )
+    return f.arrayExists(Lambda(None, ("x",), f.equals(x, rhs)), array_expr)
+
+
 def _any_attribute_filter_to_expression(
     filt: AnyAttributeFilter,
 ) -> Expression:
@@ -425,53 +513,61 @@ def trace_item_filters_to_expression(
         op = item_filter.comparison_filter.op
         v = item_filter.comparison_filter.value
 
-        # TYPE_ARRAY only supports LIKE/NOT_LIKE with string patterns — validate early.
         if k.type == AttributeKey.Type.TYPE_ARRAY:
-            if op not in (
-                ComparisonFilter.OP_LIKE,
-                ComparisonFilter.OP_NOT_LIKE,
-            ):
-                raise BadSnubaRPCRequestException(
-                    "only LIKE and NOT_LIKE comparisons are supported on array keys"
-                )
-            if v.WhichOneof("value") != "val_str":
-                raise BadSnubaRPCRequestException(
-                    "LIKE/NOT_LIKE on array keys requires a string pattern"
-                )
+            _validate_comparison_filter_type_array(op, v)
 
-        k_expression = attribute_key_to_expression(k)
+        k_expression = _trace_item_filter_key_expression(
+            attr_to_key_expression_callable=attribute_key_to_expression, key=k
+        )
 
         value_type = v.WhichOneof("value")
         if value_type is None:
             raise BadSnubaRPCRequestException("comparison does not have a right hand side")
 
-        if v.is_null:
+        if v.is_null or value_type == "val_null":
             v_expression: Expression = literal(None)
         else:
             v_expression = _attribute_value_to_expression(v)
 
         if op == ComparisonFilter.OP_EQUALS:
             _check_non_string_values_cannot_ignore_case(item_filter.comparison_filter)
-            expr = (
-                f.equals(f.lower(k_expression), f.lower(v_expression))
-                if item_filter.comparison_filter.ignore_case
-                else f.equals(k_expression, v_expression)
-            )
-            # we redefine the way equals works for nulls
-            # now null=null is true
-            expr_with_null = or_cond(expr, and_cond(f.isNull(k_expression), f.isNull(v_expression)))
-            return expr_with_null
+
+            if k.type == AttributeKey.Type.TYPE_ARRAY:
+                return _type_array_includes_scalar_expression(
+                    k_expression, v, item_filter.comparison_filter.ignore_case
+                )
+            else:
+                expr = (
+                    f.equals(f.lower(k_expression), f.lower(v_expression))
+                    if item_filter.comparison_filter.ignore_case
+                    else f.equals(k_expression, v_expression)
+                )
+                # we redefine the way equals works for nulls
+                # now null=null is true
+                expr_with_null = or_cond(
+                    expr, and_cond(f.isNull(k_expression), f.isNull(v_expression))
+                )
+                return expr_with_null
         if op == ComparisonFilter.OP_NOT_EQUALS:
             _check_non_string_values_cannot_ignore_case(item_filter.comparison_filter)
-            expr = (
-                f.notEquals(f.lower(k_expression), f.lower(v_expression))
-                if item_filter.comparison_filter.ignore_case
-                else f.notEquals(k_expression, v_expression)
-            )
-            # we redefine the way not equals works for nulls
-            # now null!=null is true
-            expr_with_null = or_cond(expr, f.xor(f.isNull(k_expression), f.isNull(v_expression)))
-            return expr_with_null
+            if k.type == AttributeKey.Type.TYPE_ARRAY:
+                return not_cond(
+                    _type_array_includes_scalar_expression(
+                        k_expression, v, item_filter.comparison_filter.ignore_case
+                    )
+                )
+            else:
+                expr = (
+                    f.notEquals(f.lower(k_expression), f.lower(v_expression))
+                    if item_filter.comparison_filter.ignore_case
+                    else f.notEquals(k_expression, v_expression)
+                )
+                # we redefine the way not equals works for nulls
+                # now null!=null is true
+                expr_with_null = or_cond(
+                    expr, f.xor(f.isNull(k_expression), f.isNull(v_expression))
+                )
+                return expr_with_null
         if op == ComparisonFilter.OP_LIKE:
             if k.type == AttributeKey.Type.TYPE_ARRAY:
                 like_fn = f.ilike if item_filter.comparison_filter.ignore_case else f.like
@@ -578,7 +674,10 @@ def trace_item_filters_to_expression(
 
     if item_filter.HasField("exists_filter"):
         return get_field_existence_expression(
-            attribute_key_to_expression(item_filter.exists_filter.key)
+            _trace_item_filter_key_expression(
+                attr_to_key_expression_callable=attribute_key_to_expression,
+                key=item_filter.exists_filter.key,
+            )
         )
 
     if item_filter.HasField("any_attribute_filter"):

snuba/web/rpc/v1/resolvers/common/aggregation.py

@@ -19,6 +19,7 @@
     Reliability,
 )
 
+from snuba.protos.common import type_array_to_stored_array_json_path
 from snuba.query.dsl import CurriedFunctions as cf
 from snuba.query.dsl import Functions as f
 from snuba.query.dsl import and_cond, column, literal
@@ -129,6 +130,9 @@ def _resolve_field_and_existence(
         else:
             raise RuntimeError("expected existence_checks to never be empty, but it is")
         return field, existence
+    elif aggregation.key.type == AttributeKey.Type.TYPE_ARRAY:
+        field = type_array_to_stored_array_json_path(aggregation.key)
+        return field, f.notEmpty(field)
     else:
         field = attribute_key_to_expression(aggregation.key)
         return field, get_field_existence_expression(field)