feat: introduce maxCollectionSize limit for materialization of collections

- Added maxCollectionSize property to EngineRuntimeState and ComputeEngine to control the maximum number of elements a collection may have when materialized.
- Default value set to 10,000, with the ability to configure it or disable the cap by setting it to Infinity or a negative number.
- Implemented logic in the Repeat and Range functions to respect the maxCollectionSize limit during materialization.
- Added tests to verify the behavior of maxCollectionSize, including edge cases for empty lists, collection types, and user-defined function broadcasting.
- Enhanced the handling of mixed-kind and mixed-dimension lists in type definitions and evaluations.
- Updated dictionary definitions to include new operators and ensure compatibility with the changes.

Author: arnog

docs/plans/2026-05-23-058-a3-lists.md

@@ -644,7 +644,57 @@ function widen2(a: Readonly<Type>, b: Readonly<Type>): Readonly<Type> {
   if (isSubtype(a, b)) return b;
   if (isSubtype(b, a)) return a;
 
-  return superType(a, b);
+  // Two types that are not subtypes of each other. Try the common
+  // supertype: this works well for related numeric types (e.g.
+  // integer/real → real). But if the supertype collapses to a generic
+  // category that loses information (e.g. 'scalar' for number+string,
+  // or 'tuple' for two tuples of different shape), surface the
+  // heterogeneity as an explicit union so downstream consumers (e.g.
+  // the List operator's type handler) can detect mixed-kind content.
+  const sup = superType(a, b);
+  if (LOSSY_SUPERTYPE.has(sup as string)) return unionTypes(a, b);
+  return sup;
+}
+
+const LOSSY_SUPERTYPE = new Set<string>([
+  'scalar',
+  'value',
+  'function',
+  'expression',
+  'collection',
+  'indexed_collection',
+  'list',
+  'set',
+  'tuple',
+  'record',
+  'dictionary',
+  'map',
+  'any',
+]);
+
+/** Build a union of two types, flattening if either is already a union and
+ *  de-duplicating identical members. Returns the simpler type if reducible.
+ */
+function unionTypes(a: Readonly<Type>, b: Readonly<Type>): Readonly<Type> {
+  const members: Type[] = [];
+  const push = (t: Readonly<Type>) => {
+    if (typeof t === 'object' && t.kind === 'union') {
+      for (const m of t.types) push(m);
+      return;
+    }
+    // de-dup by structural equality via JSON (cheap and adequate)
+    const key = typeof t === 'string' ? t : JSON.stringify(t);
+    if (
+      !members.some(
+        (m) => (typeof m === 'string' ? m : JSON.stringify(m)) === key
+      )
+    )
+      members.push(t as Type);
+  };
+  push(a);
+  push(b);
+  if (members.length === 1) return members[0];
+  return { kind: 'union', types: members };
 }
 
 /** Convert two or more types into a more specific type that is a subtype of

src/api.md

@@ -57,7 +57,7 @@ export function collectionElementType(type: Readonly<Type>): Type | undefined {
   if (type.kind === 'tuple') return widen(...type.elements.map((x) => x.type));
 
   if (type.kind === 'dictionary')
-    return parseType(`tuple<string, ${type.values}>`);
+    return parseType(`tuple<string, ${typeToString(type.values)}>`);
 
   if (type.kind === 'record') {
     return parseType(

src/common/type/subtype.ts

@@ -403,6 +403,15 @@ export function box(
   )
     return ce.number(expr);
 
+  //
+  // Box a boolean primitive as the True/False symbol.
+  // Tensors with `dtype: 'bool'` store JS booleans directly, so `.each()`
+  // and `.at()` over such a tensor need this case to yield usable
+  // symbolic values. Mirrors the `boolean → True/False` mapping in
+  // `jsValueToExpression`.
+  //
+  if (typeof expr === 'boolean') return ce.symbol(expr ? 'True' : 'False');
+
   //
   // Box a String, a Symbol or a number as a string shorthand
   //

src/common/type/utils.ts

@@ -164,7 +164,7 @@ export class BoxedDictionary
     const eltType = widen(
       ...Object.values(this._keyValues).map((op) => op.type.type)
     );
-    this._type = this.engine.type(`dictionary<${eltType}>`);
+    this._type = new BoxedType({ kind: 'dictionary', values: eltType });
     return this._type;
   }
 

src/compute-engine/boxed-expression/box.ts

@@ -1136,6 +1136,33 @@ export class BoxedFunction
         return this.engine._fn('List', results);
       }
 
+      //
+      // 2b/ Broadcast user-defined function literals over indexed collections
+      // When a function defined via `ce.assign('f', x \mapsto ...)` is applied
+      // to a list (or other finite indexed collection) and the function's
+      // parameters are scalar, map the function over the collection.
+      // Note: tuples satisfy `isFiniteIndexedCollection` and are intentionally
+      // included — a fixed-size tuple of scalars behaves like a small vector.
+      //
+      if (
+        def._isLambda &&
+        this.ops!.some((x) => isFiniteIndexedCollection(x)) &&
+        paramsAreScalar(def)
+      ) {
+        const items = zip(this._ops);
+        if (items) {
+          const results: Expression[] = [];
+          while (true) {
+            const { done, value } = items.next();
+            if (done) break;
+            results.push(
+              this.engine._fn(this.operator, value).evaluate(options)
+            );
+          }
+          return this.engine._fn('List', results);
+        }
+      }
+
       //
       // 3/ Handle evaluation of lazy collections
       //
@@ -1226,6 +1253,31 @@ export class BoxedFunction
         );
       }
 
+      //
+      // 2b/ Broadcast user-defined function literals over indexed collections.
+      // Mirrors the sync path in `_computeValue`.
+      //
+      if (
+        def?._isLambda &&
+        this.ops!.some((x) => isFiniteIndexedCollection(x)) &&
+        paramsAreScalar(def)
+      ) {
+        const items = zip(this._ops);
+        if (items) {
+          const results: Promise<Expression>[] = [];
+          while (true) {
+            const { done, value } = items.next();
+            if (done) break;
+            results.push(
+              this.engine._fn(this.operator, value).evaluateAsync(options)
+            );
+          }
+          return Promise.all(results).then((resolved) =>
+            this.engine._fn('List', resolved)
+          );
+        }
+      }
+
       //
       // 3/ Evaluate the applicable operands
       //
@@ -1570,10 +1622,98 @@ function applyFunctionLiteral(
   if (!value || value.type.isUnknown)
     return expr.engine.function(expr.operator, ops);
 
+  // Broadcast if any operand is a finite indexed collection and the
+  // function's parameter types are scalar. Zip operands and apply
+  // pointwise, returning a List of results. Tuples count as indexed
+  // collections, so a tuple of scalars also triggers broadcasting.
+  if (
+    ops.some((x) => isFiniteIndexedCollection(x)) &&
+    paramsAreScalar(value.type.type)
+  ) {
+    const items = zip(ops);
+    if (items) {
+      const results: Expression[] = [];
+      while (true) {
+        const { done, value: zipped } = items.next();
+        if (done) break;
+        results.push(apply(value, zipped).evaluate(options));
+      }
+      return expr.engine._fn('List', results);
+    }
+  }
+
   // The value is a function literal. Apply the arguments to it
   return apply(value, ops);
 }
 
+/** Returns true when every formal parameter of a signature is a scalar
+ * type (not a collection/list/tuple/function).
+ *
+ * Accepts either a `Type` (typically from a function-typed value) or a
+ * `BoxedOperatorDefinition` (whose `signature.type` is inspected).
+ *
+ * Conservative: unknown/any and non-signature types are treated as scalar,
+ * which makes this a permissive default for inferred lambda signatures.
+ * @internal
+ */
+function paramsAreScalar(source: BoxedOperatorDefinition | Type): boolean {
+  const sigType = isOperatorDefinition(source)
+    ? source.signature?.type
+    : source;
+  if (!sigType || typeof sigType === 'string') return true;
+  if (sigType.kind !== 'signature') return true;
+  const args = [
+    ...(sigType.args ?? []),
+    ...(sigType.optArgs ?? []),
+    ...(sigType.variadicArg ? [sigType.variadicArg] : []),
+  ];
+  return args.every((arg) => isScalarType(arg.type));
+}
+
+function isOperatorDefinition(
+  source: BoxedOperatorDefinition | Type
+): source is BoxedOperatorDefinition {
+  return (
+    typeof source === 'object' && source !== null && 'signature' in source
+  );
+}
+
+/** A type is "scalar" for broadcasting purposes if it is NOT a known
+ * collection-like type. Conservative: unknown/any → scalar.
+ */
+function isScalarType(t: Type): boolean {
+  if (typeof t === 'string') {
+    // String types like 'collection', 'list', 'tuple', 'set' are non-scalar.
+    if (
+      t === 'collection' ||
+      t === 'indexed_collection' ||
+      t === 'list' ||
+      t === 'tuple' ||
+      t === 'set' ||
+      t === 'dictionary' ||
+      t === 'record' ||
+      t === 'function'
+    )
+      return false;
+    return true;
+  }
+  if (
+    t.kind === 'collection' ||
+    t.kind === 'indexed_collection' ||
+    t.kind === 'list' ||
+    t.kind === 'tuple' ||
+    t.kind === 'set' ||
+    t.kind === 'dictionary' ||
+    t.kind === 'record' ||
+    t.kind === 'signature'
+  )
+    return false;
+  if (t.kind === 'union' || t.kind === 'intersection')
+    return t.types.every((x) => isScalarType(x));
+  if (t.kind === 'negation') return isScalarType(t.type);
+  return true;
+}
+
 /**  Eagerly evaluate xs by iterating over its elements.
  *
  * If eager is true, evaluate DEFAULT_MATERIALIZATION elements.
@@ -1598,6 +1738,14 @@ function materialize(
   const isIndexed = expr.isIndexedCollection;
   const isFinite = expr.isFiniteCollection;
 
+  // Leave oversized indexed collections in their lazy form. Consumers
+  // can detect the size via `.count` without risking OOM.
+  if (isIndexed && isFinite) {
+    const count = expr.count;
+    if (count !== undefined && count > expr.engine.maxCollectionSize)
+      return expr;
+  }
+
   const xs: Expression[] = [];
 
   if (!expr.isEmptyCollection) {

src/compute-engine/boxed-expression/boxed-dictionary.ts

@@ -86,6 +86,13 @@ export class _BoxedOperatorDefinition implements BoxedOperatorDefinition {
   signature: BoxedType;
   inferredSignature = true;
 
+  /** True if this operator definition was created from a user-defined
+   * function literal (e.g. via `ce.assign('f', ce.parse('x \\mapsto x^2'))`).
+   * Used to enable auto-broadcasting when applied to indexed collections.
+   * @internal
+   */
+  _isLambda = false;
+
   type?: (
     ops: ReadonlyArray<Expression>,
     options: { engine: ComputeEngine }
@@ -336,6 +343,11 @@ export class _BoxedOperatorDefinition implements BoxedOperatorDefinition {
         );
       }
 
+      // Mark this operator definition as backed by a user-defined function
+      // literal. Enables auto-broadcasting at apply time.
+      if (isFunction(boxedFn) && boxedFn.operator === 'Function')
+        this._isLambda = true;
+
       const fn = applicable(boxedFn);
       evaluate = (xs, _options) => fn(xs);
       Object.defineProperty(evaluate, 'toString', {

src/compute-engine/boxed-expression/boxed-function.ts

@@ -438,9 +438,34 @@ export function expressionTensorInfo(
         }
       }
     }
-    // 4b. all leaves → accumulate dtype
+    // 4b. all leaves → accumulate dtype.
+    // Reject collection-like leaves and strings so mixed-kind lists like
+    // [1, 'hello'] and mixed-dim point lists like [Tuple(1,2), Tuple(3,4,5)]
+    // aren't misidentified as tensors — the List operator's type handler
+    // relies on this to surface a heterogeneous element type.
     else {
       for (const item of t) {
+        // Operator-name check: tensor detection runs on raw boxed ops where
+        // item.type may still be 'unknown', so we can't rely on type inspection.
+        const op = item.operator;
+        if (
+          op === 'Tuple' ||
+          op === 'Pair' ||
+          op === 'Single' ||
+          op === 'Triple' ||
+          op === 'Quadruple' ||
+          op === 'KeyValuePair' ||
+          op === 'Dictionary' ||
+          op === 'Set' ||
+          op === 'Record'
+        ) {
+          valid = false;
+          return;
+        }
+        if (item.type.type === 'string') {
+          valid = false;
+          return;
+        }
         dtype = getSupertype(dtype, getExpressionDatatype(item));
       }
     }

src/compute-engine/boxed-expression/boxed-operator-definition.ts

@@ -79,9 +79,7 @@ export function extractIntervalBounds(
     // Less(a, b, c)   means  a < b < c
     // Greater(a, b)   means  a > b  → treated as b < a  (flipped to [b, a])
     const flipped =
-      op === 'Greater' || op === 'GreaterEqual'
-        ? [...ops].reverse()
-        : ops;
+      op === 'Greater' || op === 'GreaterEqual' ? [...ops].reverse() : ops;
 
     // Walk the (flipped) chain looking for `symbol` as an operand.
     // For chain [lower, symbol]: lower bound.
@@ -121,10 +119,7 @@ export function extractIntervalBounds(
 
 function _mergeBounds(into: IntervalBounds, from: IntervalBounds): void {
   if (from.lower !== undefined) {
-    if (
-      into.lower === undefined ||
-      from.lower.isGreater(into.lower) === true
-    ) {
+    if (into.lower === undefined || from.lower.isGreater(into.lower) === true) {
       into.lower = from.lower;
       into.lowerStrict = from.lowerStrict;
     } else if (from.lower.isSame(into.lower)) {
@@ -133,10 +128,7 @@ function _mergeBounds(into: IntervalBounds, from: IntervalBounds): void {
     }
   }
   if (from.upper !== undefined) {
-    if (
-      into.upper === undefined ||
-      from.upper.isLess(into.upper) === true
-    ) {
+    if (into.upper === undefined || from.upper.isLess(into.upper) === true) {
       into.upper = from.upper;
       into.upperStrict = from.upperStrict;
     } else if (from.upper.isSame(into.upper)) {
@@ -239,10 +231,7 @@ export function getInequalityBoundsFromAssumptions(
     // Case 3: symbol < 0 => symbol has upper bound 0
     if (isSymbol(lhs, symbol)) {
       const bound = ce.Zero;
-      if (
-        result.upper === undefined ||
-        bound.isLess(result.upper) === true
-      ) {
+      if (result.upper === undefined || bound.isLess(result.upper) === true) {
         result.upper = bound;
         result.upperStrict = isStrict;
       }
@@ -271,10 +260,7 @@ export function getInequalityBoundsFromAssumptions(
       if (hasSymbol && constantSum !== 0) {
         // symbol + k < 0 => symbol < -k
         const bound = ce.expr(-constantSum);
-        if (
-          result.upper === undefined ||
-          bound.isLess(result.upper) === true
-        ) {
+        if (result.upper === undefined || bound.isLess(result.upper) === true) {
           result.upper = bound;
           result.upperStrict = isStrict;
         }