rework fusion framework based on immutable named types

This commit reworks downcast, upcast, and fuser to presume
that named types are immutable.  The changes for immutability
are forthcoming but the fusion code now assumes this invariant.

This new deisng involved fused types to now carry the named types
instead of stripping them and deferring them to the fusion
subtypes.  This is key to allow the fusion runtime to manipulate
named types. When support for recursive types is added,
the recursive type will remain in the fusion and the concrete types
will be fused below.  With this approach, a fused recursive type will
rarely need to be unfused.

This commit also tightens up the algorithms for upcast/downcast
so they should now be generally more reliable and can serve as
a model for implementing their vam counterparts.

Author: mccanne

runtime/sam/expr/agg/fuser.go

@@ -121,6 +121,15 @@ func (f *Fuser) fuse(a, b super.Type) super.Type {
 		}
 	case *super.TypeNamed:
 		if b, ok := b.(*super.TypeNamed); ok && a.Name == b.Name {
+			if a.Type != b.Type {
+				// The fusion algorithm does not handle named types that change.
+				// We will soon maked such types immutable, but for now we just
+				// return type error({}) to avoid any tests that might do this.
+				recType := f.sctx.MustLookupTypeRecord([]super.Field{
+					super.NewField(a.Name, a.Type),
+				})
+				return f.sctx.LookupTypeError(recType)
+			}
 			named, err := f.sctx.LookupTypeNamed(a.Name, f.fuse(a.Type, b.Type))
 			if err != nil {
 				panic(err)

runtime/sam/expr/function/defuse.go

@@ -16,7 +16,7 @@ type defuse struct {
 func NewDefuse(sctx *super.Context) *defuse {
 	return &defuse{
 		sctx:     sctx,
-		downcast: &downcast{sctx: sctx},
+		downcast: &downcast{sctx: sctx, name: "defuse"},
 		has:      make(map[super.Type]bool),
 	}
 }
@@ -95,11 +95,11 @@ func (d *defuse) eval(in super.Value) super.Value {
 	case *super.TypeUnion:
 		return d.eval(in.DeunionIntoNameds())
 	case *super.TypeFusion:
-		_, subType := typ.Deref(d.sctx, in.Bytes())
-		if out, ok := d.downcast.Cast(in, subType); ok {
-			return out
+		out, errVal := d.downcast.defuse(typ, in.Bytes())
+		if errVal != nil {
+			return *errVal
 		}
-		return d.sctx.WrapError("cannot defuse super value", in)
+		return out
 	default:
 		// primitives, named types, enums
 		// BTW, named types are a barrier to defuse.

runtime/sam/expr/function/downcast.go

@@ -1,17 +1,20 @@
 package function
 
 import (
+	"slices"
+
 	"github.com/brimdata/super"
 	"github.com/brimdata/super/scode"
 	"github.com/brimdata/super/sup"
 )
 
 type downcast struct {
 	sctx *super.Context
+	name string
 }
 
-func NewDowncast(sctx *super.Context) Caster {
-	return &downcast{sctx}
+func NewDowncast(sctx *super.Context, name string) Caster {
+	return &downcast{sctx, name}
 }
 
 func (d *downcast) Call(args []super.Value) super.Value {
@@ -23,164 +26,238 @@ func (d *downcast) Call(args []super.Value) super.Value {
 	if err != nil {
 		panic(err)
 	}
-	val, ok := d.Cast(from, typ)
-	if !ok {
-		return d.sctx.WrapError("downcast: value not a supertype of "+sup.FormatType(typ), from)
+	val, errVal := d.downcast(from.Type(), from.Bytes(), typ)
+	if errVal != nil {
+		return *errVal
 	}
 	return val
 }
 
 func (d *downcast) Cast(from super.Value, to super.Type) (super.Value, bool) {
-	var b scode.Builder
-	if ok := d.downcast(&b, from.Type(), from.Bytes(), to); ok {
-		return super.NewValue(to, b.Bytes().Body()), true
-	}
-	return super.Value{}, false
+	val, errVal := d.downcast(from.Type(), from.Bytes(), to)
+	return val, errVal == nil
 }
 
-func (d *downcast) downcast(b *scode.Builder, typ super.Type, bytes scode.Bytes, to super.Type) bool {
-	typ, bytes = deunion(typ, bytes)
-	if superType, ok := typ.(*super.TypeFusion); ok {
-		superBytes, _ := superType.Deref(d.sctx, bytes)
-		return d.downcast(b, superType.Type, superBytes, to)
+func (d *downcast) downcast(typ super.Type, bytes scode.Bytes, to super.Type) (super.Value, *super.Value) {
+	if _, ok := to.(*super.TypeUnion); !ok {
+		if fusionType, ok := typ.(*super.TypeFusion); ok {
+			superBytes, subtype := fusionType.Deref(d.sctx, bytes)
+			return d.downcast(fusionType.Type, superBytes, subtype)
+		}
 	}
-	typ = super.TypeUnder(typ)
+	typ, bytes = deunion(typ, bytes)
 	switch to := to.(type) {
 	case *super.TypeRecord:
-		return d.toRecord(b, typ, bytes, to)
+		return d.toRecord(typ, bytes, to)
 	case *super.TypeArray:
-		return d.toArray(b, typ, bytes, to)
+		return d.toArray(typ, bytes, to)
 	case *super.TypeSet:
-		return d.toSet(b, typ, bytes, to)
+		return d.toSet(typ, bytes, to)
 	case *super.TypeMap:
-		return d.toMap(b, typ, bytes, to)
+		return d.toMap(typ, bytes, to)
 	case *super.TypeUnion:
-		return d.toUnion(b, typ, bytes, to)
+		return d.toUnion(typ, bytes, to)
 	case *super.TypeError:
-		return d.toError(b, typ, bytes, to)
+		return d.toError(typ, bytes, to)
 	case *super.TypeNamed:
-		return d.downcast(b, typ, bytes, to.Type)
+		return d.toNamed(typ, bytes, to)
 	case *super.TypeFusion:
 		// Can't downcast to a super type
-		return false
+		return super.Value{}, d.sctx.WrapError("downcast: cannot downcast to a fusion type", super.NewValue(typ, bytes)).Ptr()
 	default:
 		if typ == to {
-			b.Append(bytes)
-			return true
+			return super.NewValue(typ, bytes), nil
+		} else {
+			typ, bytes := deunion(typ, bytes)
+			if typ == to {
+				return super.NewValue(typ, bytes), nil
+			}
 		}
-		return false
+		return super.Value{}, d.errMismatch(typ, bytes, to)
 	}
 }
 
-func (d *downcast) toRecord(b *scode.Builder, typ super.Type, bytes scode.Bytes, to *super.TypeRecord) bool {
+func (d *downcast) defuse(fusionType *super.TypeFusion, bytes scode.Bytes) (super.Value, *super.Value) {
+	superBytes, subtype := fusionType.Deref(d.sctx, bytes)
+	return d.downcast(fusionType.Type, superBytes, subtype)
+}
+
+func (d *downcast) toRecord(typ super.Type, bytes scode.Bytes, to *super.TypeRecord) (super.Value, *super.Value) {
 	fromType, ok := typ.(*super.TypeRecord)
 	if !ok {
-		return false
+		return super.Value{}, d.errMismatch(typ, bytes, to)
 	}
 	var nones []int
 	var optOff int
+	b := scode.NewBuilder()
 	b.BeginContainer()
-	for _, toField := range to.Fields { // ranging through to fields and lookup up from
+	for k, toField := range to.Fields { // ranging through to fields and lookup up from
 		elemType, elemBytes, none, ok := derefWithNoneAndOk(fromType, bytes, toField.Name)
 		if !ok {
 			// The super value must have all the fields of the subtype cast.
 			// It's missing a field, so fail.
-			return false
+			return super.Value{}, d.errSubtype(typ, bytes, to)
 		}
 		if none {
 			if !toField.Opt {
 				// A none can't go in a non-optional field.
-				return false
+				return super.Value{}, d.errSubtype(typ, bytes, to)
 			}
 			nones = append(nones, optOff)
 			optOff++
+		} else if toField.Opt && !fromType.Fields[k].Opt {
+			return super.Value{}, d.errSubtype(typ, bytes, to)
 		} else {
 			// We have the value and the to field.  Downcast recursively.
-			if ok := d.downcast(b, elemType, elemBytes, toField.Type); !ok {
-				return false
+			val, errVal := d.downcast(elemType, elemBytes, toField.Type)
+			if errVal != nil {
+				return super.Value{}, errVal
 			}
 			if toField.Opt {
 				optOff++
 			}
+			b.Append(val.Bytes())
 		}
 	}
 	b.EndContainerWithNones(to.Opts, nones)
-	return true
+	return super.NewValue(to, b.Bytes().Body()), nil
 }
 
-func (d *downcast) toArray(b *scode.Builder, typ super.Type, bytes scode.Bytes, to *super.TypeArray) bool {
+func (d *downcast) toArray(typ super.Type, bytes scode.Bytes, to *super.TypeArray) (super.Value, *super.Value) {
 	if arrayType, ok := typ.(*super.TypeArray); ok {
-		return d.toContainer(b, arrayType.Type, bytes, to.Type)
+		return d.toContainer(arrayType.Type, bytes, to, to.Type)
 	}
-	return false
+	return super.Value{}, d.errMismatch(typ, bytes, to)
 }
 
-func (d *downcast) toSet(b *scode.Builder, typ super.Type, bytes scode.Bytes, to *super.TypeSet) bool {
+func (d *downcast) toSet(typ super.Type, bytes scode.Bytes, to *super.TypeSet) (super.Value, *super.Value) {
 	if setType, ok := typ.(*super.TypeSet); ok {
 		// XXX normalize set contents? can reach into body here blah
-		return d.toContainer(b, setType.Type, bytes, to.Type)
+		return d.toContainer(setType.Type, bytes, to, to.Type)
 	}
-	return false
+	return super.Value{}, d.errMismatch(typ, bytes, to)
 }
 
-func (d *downcast) toContainer(b *scode.Builder, typ super.Type, bytes scode.Bytes, to super.Type) bool {
+func (d *downcast) toContainer(elemType super.Type, bytes scode.Bytes, to super.Type, toElem super.Type) (super.Value, *super.Value) {
+	b := scode.NewBuilder()
 	b.BeginContainer()
 	for it := bytes.Iter(); !it.Done(); {
-		if ok := d.downcast(b, typ, it.Next(), to); !ok {
-			return false
+		val, errVal := d.downcast(elemType, it.Next(), toElem)
+		if errVal != nil {
+			return super.Value{}, errVal
 		}
+		b.Append(val.Bytes())
 	}
 	b.EndContainer()
-	return true
+	return super.NewValue(to, b.Bytes().Body()), nil
 }
 
-func (d *downcast) toMap(b *scode.Builder, typ super.Type, bytes scode.Bytes, to *super.TypeMap) bool {
+func (d *downcast) toMap(typ super.Type, bytes scode.Bytes, to *super.TypeMap) (super.Value, *super.Value) {
 	mapType, ok := typ.(*super.TypeMap)
 	if !ok {
-		return false
+		return super.Value{}, d.errMismatch(typ, bytes, to)
 	}
+	b := scode.NewBuilder()
 	b.BeginContainer()
 	for it := bytes.Iter(); !it.Done(); {
-		if ok := d.downcast(b, mapType.KeyType, it.Next(), to.KeyType); !ok {
-			return false
+		key, errVal := d.downcast(mapType.KeyType, it.Next(), to.KeyType)
+		if errVal != nil {
+			return super.Value{}, errVal
 		}
-		if ok := d.downcast(b, mapType.ValType, it.Next(), to.ValType); !ok {
-			return false
+		b.Append(key.Bytes())
+		val, errVal := d.downcast(mapType.ValType, it.Next(), to.ValType)
+		if errVal != nil {
+			return super.Value{}, errVal
 		}
+		b.Append(val.Bytes())
 	}
 	b.EndContainer()
-	return true
+	return super.NewValue(to, b.Bytes().Body()), nil
 }
 
-func (d *downcast) toUnion(b *scode.Builder, typ super.Type, bytes scode.Bytes, to *super.TypeUnion) bool {
-	tag := d.subTypeOf(typ, bytes, to.Types)
+func (d *downcast) toUnion(typ super.Type, bytes scode.Bytes, to *super.TypeUnion) (super.Value, *super.Value) {
+	if typ == to {
+		return super.NewValue(typ, bytes), nil
+	}
+	tag, typ, bytes := d.subTypeOf(typ, bytes, to.Types)
 	if tag < 0 {
-		return false
+		if _, ok := typ.(*super.TypeUnion); ok {
+			typ, bytes = deunion(typ, bytes)
+			return d.downcast(typ, bytes, to)
+		}
+		return super.Value{}, d.errSubtype(typ, bytes, to)
 	}
-	super.BeginUnion(b, tag)
-	if ok := d.downcast(b, typ, bytes, to.Types[tag]); !ok {
-		return false
+	val, errVal := d.downcast(typ, bytes, to.Types[tag])
+	if errVal != nil {
+		return super.Value{}, errVal
 	}
+	b := scode.NewBuilder()
+	super.BeginUnion(b, tag)
+	b.Append(val.Bytes())
 	b.EndContainer()
-	return true
+	return super.NewValue(to, b.Bytes().Body()), nil
 }
 
-func (d *downcast) toError(b *scode.Builder, typ super.Type, bytes scode.Bytes, to *super.TypeError) bool {
+// subTypeOf finds the tag in the union array types that this value should be
+// downcast to.  If the child value is a fusion value, then the type must match
+// the subtype of the fusion value.  Otherwise, the child wasn't fused, and by
+// definition of a fusion type, one of the union types must exactly match the
+// child type.
+func (d *downcast) subTypeOf(typ super.Type, bytes scode.Bytes, types []super.Type) (int, super.Type, []byte) {
+	if fusionType, ok := typ.(*super.TypeFusion); ok {
+		superBytes, subtype := fusionType.Deref(d.sctx, bytes)
+		return slices.Index(types, subtype), fusionType.Type, superBytes
+	}
+	return slices.Index(types, typ), typ, bytes
+}
+
+func (d *downcast) toError(typ super.Type, bytes scode.Bytes, to *super.TypeError) (super.Value, *super.Value) {
 	if errorType, ok := typ.(*super.TypeError); ok {
-		return d.downcast(b, errorType.Type, bytes, to.Type)
+		body, errVal := d.downcast(errorType.Type, bytes, to.Type)
+		if errVal != nil {
+			return super.Value{}, errVal
+		}
+		return super.NewValue(to, body.Bytes()), nil
 	}
-	return false
+	return super.Value{}, d.errMismatch(typ, bytes, to)
 }
 
-func (d *downcast) subTypeOf(typ super.Type, bytes scode.Bytes, types []super.Type) int {
-	// XXX TBD we should make a subtype() function that returns true if a type is
-	// a subtype of another and use that here and expose it to the language.
-	var dummy scode.Builder
-	for k, t := range types {
-		if ok := d.downcast(&dummy, typ, bytes, t); ok {
-			return k
+func (d *downcast) toNamed(typ super.Type, bytes scode.Bytes, to *super.TypeNamed) (super.Value, *super.Value) {
+	if unionType, ok := typ.(*super.TypeUnion); ok {
+		typ, bytes = deunion(typ, bytes)
+		// If we are casting a union type to a named, we need to look through the
+		// union for the named type in question since type fusion fuses named
+		// types by name.  Then when we find the name, we need to form the subtype
+		// from the union options present.
+		for _, t := range unionType.Types {
+			if named, ok := t.(*super.TypeNamed); ok && named.Name == to.Name {
+				typ, bytes = deunion(typ, bytes)
+				return super.NewValue(to, bytes), nil
+			}
 		}
-		dummy.Reset()
+		return super.Value{}, d.errMismatch(typ, bytes, to)
 	}
-	return -1
+	if fromType, ok := typ.(*super.TypeNamed); ok {
+		if fromType.Name != to.Name {
+			return super.Value{}, d.errMismatch(typ, bytes, to)
+		}
+		val, errVal := d.downcast(fromType.Type, bytes, to.Type)
+		if errVal != nil {
+			return super.Value{}, errVal
+		}
+		return super.NewValue(to, val.Bytes()), errVal
+	}
+	val, errVal := d.downcast(typ, bytes, to.Type)
+	if errVal != nil {
+		return super.Value{}, errVal
+	}
+	return super.NewValue(to, val.Bytes()), errVal
+}
+
+func (d *downcast) errMismatch(typ super.Type, bytes []byte, to super.Type) *super.Value {
+	return d.sctx.WrapError("downcast: type mismatch to "+sup.FormatType(to), super.NewValue(typ, bytes)).Ptr()
+}
+
+func (d *downcast) errSubtype(typ super.Type, bytes []byte, to super.Type) *super.Value {
+	return d.sctx.WrapError("downcast: invalid subtype "+sup.FormatType(to), super.NewValue(typ, bytes)).Ptr()
 }

runtime/sam/expr/function/function.go

@@ -58,7 +58,7 @@ func New(sctx *super.Context, name string, narg int) (expr.Function, error) {
 	case "downcast":
 		argmin = 2
 		argmax = 2
-		f = &downcast{sctx}
+		f = &downcast{sctx: sctx, name: "downcast"}
 	case "error":
 		f = &Error{sctx: sctx}
 	case "fields":

runtime/sam/expr/function/fusion.go

@@ -14,7 +14,7 @@ type fusion struct {
 func newFusion(sctx *super.Context) *fusion {
 	return &fusion{
 		sctx:     sctx,
-		downcast: NewDowncast(sctx),
+		downcast: NewDowncast(sctx, "fusion"),
 	}
 }