feat: add support for unresolved expressions

Author: malinjawi

dialects/tests/expressions_test.yaml

@@ -7,6 +7,7 @@ supported_expressions:
   - LITERAL
   - SELECTION
   - SCALAR_FUNCTION
+  - expression: NAMED_EXPRESSION
   - expression: CAST
     metadata:
       truncates_on_overflow: true

dialects/tests/types_test.yaml

@@ -9,6 +9,7 @@ dependencies:
 supported_types:
   - I8
   - I16
+  - UNKNOWN
   - type: I32
     metadata:
       storage_bytes: 4

grammar/FuncTestCaseParser.g4

@@ -242,6 +242,7 @@ scalarType
   | dateType                             #date
   | intervalYearType                     #intervalYear
   | UUID isnull=QMark?                   #uuid
+  | Unknown isnull=QMark?                #unknown
   | UserDefined Identifier isnull=QMark? #userDefined
   ;
 

grammar/SubstraitLexer.g4

@@ -116,6 +116,8 @@ Number
   : '-'? Int
   ;
 
+Unknown: 'UNKNOWN';
+
 Identifier
   : ('A'..'Z' | '_' | '$') ('A'..'Z' | '_' | '$' | Digit)*
   ;

grammar/SubstraitType.g4

@@ -23,6 +23,7 @@ scalarType
   | Date                    #date
   | Interval_Year           #intervalYear
   | UUID                    #uuid
+  | Unknown                 #unknown
   ;
 
 parameterizedType

proto/substrait/algebra.proto

@@ -1019,6 +1019,7 @@ message Expression {
     Lambda lambda = 15;
     LambdaInvocation lambda_invocation = 16;
     ExecutionContextVariable execution_context_variable = 17;
+    NamedExpression named_expression = 18;
 
     // deprecated: enum literals are only sensible in the context of
     // function arguments, for which FunctionArgument should now be
@@ -1254,6 +1255,17 @@ message Expression {
     Nested.Struct arguments = 2;
   }
 
+  // A named expression that is not yet resolved to a positional reference or
+  // concrete expression. The names field is used to represent namespacing
+  // (e.g. qualifier and field name). Producers and consumers must share enough
+  // external context to resolve this expression before execution unless the
+  // consumer defines semantics for unresolved named expressions. Until
+  // resolved, a NamedExpression's type is Type.Unknown.
+  message NamedExpression {
+    repeated string names = 1;
+    substrait.extensions.AdvancedExtension advanced_extension = 10;
+  }
+
   // A scalar function call.
   message ScalarFunction {
     // Points to a function_anchor defined in this plan, which must refer
@@ -1271,6 +1283,9 @@ message Expression {
     //  - Enum arguments must be bound using FunctionArgument.enum
     //    followed by Enum.specified, with a string that case-insensitively
     //    matches one of the allowed options.
+    //
+    // In a partially bound expression, Type.Unknown may stand in for any
+    // concrete type while binding value or type arguments.
     repeated FunctionArgument arguments = 4;
 
     // Options to specify behavior for corner cases, or leave behavior
@@ -1279,7 +1294,8 @@ message Expression {
     repeated FunctionOption options = 5;
 
     // Must be set to the return type of the function, exactly as derived
-    // using the declaration in the extension.
+    // using the declaration in the extension. In a partially bound expression,
+    // this may be Type.Unknown if the concrete return type is unresolved.
     Type output_type = 3;
 
     // Deprecated; use arguments instead.
@@ -1310,6 +1326,9 @@ message Expression {
     //  - Enum arguments must be bound using FunctionArgument.enum
     //    followed by Enum.specified, with a string that case-insensitively
     //    matches one of the allowed options.
+    //
+    // In a partially bound expression, Type.Unknown may stand in for any
+    // concrete type while binding value or type arguments.
     repeated FunctionArgument arguments = 9;
 
     // Options to specify behavior for corner cases, or leave behavior
@@ -1318,7 +1337,8 @@ message Expression {
     repeated FunctionOption options = 11;
 
     // Must be set to the return type of the function, exactly as derived
-    // using the declaration in the extension.
+    // using the declaration in the extension. In a partially bound expression,
+    // this may be Type.Unknown if the concrete return type is unresolved.
     Type output_type = 7;
 
     // Describes which part of the window function to perform within the
@@ -1839,6 +1859,9 @@ message AggregateFunction {
   //  - Optional enum arguments must be bound using FunctionArgument.enum
   //    followed by either Enum.specified or Enum.unspecified. If specified,
   //    the string must case-insensitively match one of the allowed options.
+  //
+  // In a partially bound expression, Type.Unknown may stand in for any
+  // concrete type while binding value or type arguments.
   repeated FunctionArgument arguments = 7;
 
   // Options to specify behavior for corner cases, or leave behavior
@@ -1847,7 +1870,8 @@ message AggregateFunction {
   repeated FunctionOption options = 8;
 
   // Must be set to the return type of the function, exactly as derived
-  // using the declaration in the extension.
+  // using the declaration in the extension. In a partially bound expression,
+  // this may be Type.Unknown if the concrete return type is unresolved.
   Type output_type = 5;
 
   // Describes which part of the aggregation to perform within the context of

proto/substrait/type.proto

@@ -46,6 +46,13 @@ message Type {
     Map map = 28;
     Func func = 38;
 
+    // A placeholder type whose concrete type has not yet been resolved.
+    // Unknown may be used where a concrete type would normally appear in
+    // partially bound plans and expressions, and must be resolved to a
+    // concrete type before execution unless the consumer defines semantics for
+    // unknown-typed values.
+    Unknown unknown = 39;
+
     UserDefined user_defined = 30;
 
     // Deprecated in favor of user_defined, which allows nullability and
@@ -225,6 +232,12 @@ message Type {
     Nullability nullability = 3;
   }
 
+  message Unknown {
+    // Optional nullability constraint for this unknown type. If unspecified,
+    // both the concrete type and its nullability are unresolved.
+    Nullability nullability = 1;
+  }
+
   message UserDefined {
     // References a type_anchor defined in the plan's extension declarations.
     uint32 type_reference = 1;

site/docs/expressions/_config

@@ -1,5 +1,6 @@
 arrange:
   - field_references.md
+  - unbound_expressions.md
   - scalar_functions.md
   - aggregate_functions.md
   - specialized_record_expressions.md

site/docs/expressions/extended_expression.md

@@ -23,3 +23,11 @@ For a message with multiple expressions, users may produce each Extended Express
 ## Function extensions
 
 Function extensions work the same for both Extended Expression and the original Expression defined in the Substrait protocol.
+
+## Partially Bound Expressions
+
+Extended Expression can also carry partially bound expressions for producers that do not yet know the full input schema. In this form, field-like references can be represented as `NamedExpression`, and any known-but-unresolved schema or expression types can use `unknown`. A consumer must bind these names and types before execution unless it defines its own unresolved-expression semantics.
+
+```protobuf
+--8<-- "examples/proto-textformat/extended_expression/unbound_named_projection.textproto"
+```

site/docs/expressions/unbound_expressions.md

@@ -0,0 +1,54 @@
+# Unbound Expressions
+
+Substrait normally represents bound relational expressions: field references are positional, value types are known, and function invocations identify implementations whose argument and return types have been derived. Some producers need to serialize expressions earlier in planning, before names and types have been resolved.
+
+An expression tree is partially bound when it contains either an [`unknown`](../types/type_classes.md#unknown-type) type or a `NamedExpression`. Consumers may validate and transform partially bound expressions, but must resolve them before execution unless they define their own semantics for unresolved names or unknown-typed values.
+
+This applies anywhere Substrait uses `Expression`, including both full `Plan` messages and `ExtendedExpression`. The examples below use `ExtendedExpression` because expression-only filter / projection APIs are a common motivating use case, not because partially bound expressions are limited to that entry point.
+
+## Detecting Partial Binding
+
+There is no separate "unbound expression" message. Instead, partially bound state is detected structurally:
+
+- If any expression, function argument, or schema field type is `unknown`, the expression is partially bound.
+- If any expression contains `NamedExpression`, the expression is partially bound.
+
+This is the canonical way to distinguish fully bound expressions from partially bound expressions in Substrait.
+
+## Unknown Type
+
+The `unknown` type marks an expression whose concrete type is not known yet. It may be used anywhere a concrete type would normally be expected in a partially bound function call. If only the nullability is known, set the nullability field; otherwise leave it unspecified.
+
+## Named Expression
+
+`NamedExpression` represents a reference by name instead of ordinal position. The `names` field stores one or more namespace components, such as `["foo"]` for an unqualified name or `["orders", "amount"]` for a qualified name. Until resolved, a named expression's type is `unknown`. Resolution of these components is intentionally external to Substrait and must be understood by both producer and consumer.
+
+```protobuf
+--8<-- "examples/proto-textformat/unbound_expression/named_expression.textproto"
+```
+
+## Function Example
+
+Partially bound function calls can use named expressions as value arguments and `unknown` as the output type when the return type cannot be derived yet.
+
+```protobuf
+--8<-- "examples/proto-textformat/unbound_expression/scalar_function_unknown.textproto"
+```
+
+## Extended Expression Protocols
+
+Expression-level APIs, such as filters and projections exchanged outside a full `Plan`, should use `ExtendedExpression`. This lets the producer include output names, any known input names, and extension declarations next to the expression tree.
+
+If the input names are known but their types are not, `base_schema` can contain fields with `unknown` types. If the function overload is also unresolved, the function can refer to the `extension:io.substrait:unknown` extension until a downstream binder replaces it with a concrete function reference and concrete output type. In that case, the referenced function name should use the normal Substrait function-signature form with `unknown` short names, such as `add:unknown_unknown`.
+
+```protobuf
+--8<-- "examples/proto-textformat/extended_expression/unbound_named_projection.textproto"
+```
+
+Consumers that execute expressions must reject or bind away all `NamedExpression` and `unknown` types before execution unless they explicitly support unresolved semantics. A typical binder resolves `NamedExpression` values to `FieldReference`, replaces `unknown` input and output types with concrete types, and updates unresolved function references to concrete overloads.
+
+## Plans, ReadRel, and Dynamic Parameters
+
+Partially bound expressions may also appear inside a `Plan`. In that case, relational operators such as `ReadRel` still provide the surrounding schema context. If field names are known, `ReadRel.base_schema` remains the source of those names even before ordinal binding has happened. A downstream binder can use that schema to resolve `NamedExpression` values into positional `FieldReference`s. If names are known but types are not, `base_schema` can use `unknown` types until type resolution completes.
+
+`NamedExpression` is intentionally distinct from `DynamicParameter`. A `DynamicParameter` represents an externally supplied literal value, identified by `parameter_anchor` and resolved through `DynamicParameterBinding`. A `NamedExpression` represents an unresolved field-like or catalog-like reference that is expected to bind against schema or catalog context. Because these are different binding problems, this proposal keeps `DynamicParameterBinding` and `NamedExpression` separate.