[Custom Descriptors] Heap2Local: Handle definitely-failing descriptor casts (#7919)

A definitely-failing ref.cast_desc is marked as non-escaping. We
need to handle that, like in normal casts, by trapping rather than allowing
the allocation to flow onward.

Author: kripken

src/passes/Heap2Local.cpp

@@ -890,17 +890,28 @@ struct Struct2Local : PostWalker<Struct2Local> {
         }
       } else {
         assert(allocIsCastRef);
-        // The cast succeeds iff the optimized allocation's descriptor is the
-        // same as the given descriptor and traps otherwise.
-        auto type = allocation->desc->type;
-        replaceCurrent(builder.blockify(
-          builder.makeDrop(curr->ref),
-          builder.makeIf(
-            builder.makeRefEq(
-              curr->desc,
-              builder.makeLocalGet(localIndexes[fields.size()], type)),
-            builder.makeRefNull(allocation->type.getHeapType()),
-            builder.makeUnreachable())));
+        if (!Type::isSubType(allocation->type, curr->type)) {
+          // The cast fails, so it must trap. We mark such failing casts as
+          // fully consuming their inputs, so we cannot just emit the explicit
+          // descriptor equality check below because it would appear to be able
+          // to propagate the optimized allocation on to the parent (as a null
+          // value, which might not validate).
+          replaceCurrent(builder.blockify(builder.makeDrop(curr->ref),
+                                          builder.makeDrop(curr->desc),
+                                          builder.makeUnreachable()));
+        } else {
+          // The cast succeeds iff the optimized allocation's descriptor is the
+          // same as the given descriptor and traps otherwise.
+          auto type = allocation->desc->type;
+          replaceCurrent(builder.blockify(
+            builder.makeDrop(curr->ref),
+            builder.makeIf(
+              builder.makeRefEq(
+                curr->desc,
+                builder.makeLocalGet(localIndexes[fields.size()], type)),
+              builder.makeRefNull(allocation->type.getHeapType()),
+              builder.makeUnreachable())));
+        }
       }
     } else {
       // We know this RefCast receives our allocation, so we can see whether it

test/lit/passes/heap2local-desc.wast

@@ -898,3 +898,51 @@
   )
 )
 
+;; A definitely-failing descriptor cast. We have two pairs of descriptor/
+;; describee, and create an $A2 that we try to cast to the unrelated $A.
+(module
+  (rec
+    ;; CHECK:      (rec
+    ;; CHECK-NEXT:  (type $A (descriptor $B (struct)))
+    (type $A (descriptor $B (struct)))
+    ;; CHECK:       (type $B (describes $A (struct)))
+    (type $B (describes $A (struct)))
+
+    ;; CHECK:       (type $A2 (descriptor $B2 (struct)))
+    (type $A2 (descriptor $B2 (struct)))
+    ;; CHECK:       (type $B2 (describes $A2 (struct)))
+    (type $B2 (describes $A2 (struct)))
+  )
+
+  ;; CHECK:      (type $4 (func (result (ref $A))))
+
+  ;; CHECK:      (func $A (type $4) (result (ref $A))
+  ;; CHECK-NEXT:  (local $0 (ref (exact $B2)))
+  ;; CHECK-NEXT:  (local $1 (ref (exact $B2)))
+  ;; CHECK-NEXT:  (drop
+  ;; CHECK-NEXT:   (block (result nullref)
+  ;; CHECK-NEXT:    (local.set $1
+  ;; CHECK-NEXT:     (struct.new_default $B2)
+  ;; CHECK-NEXT:    )
+  ;; CHECK-NEXT:    (local.set $0
+  ;; CHECK-NEXT:     (local.get $1)
+  ;; CHECK-NEXT:    )
+  ;; CHECK-NEXT:    (ref.null none)
+  ;; CHECK-NEXT:   )
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT:  (drop
+  ;; CHECK-NEXT:   (block (result nullref)
+  ;; CHECK-NEXT:    (ref.null none)
+  ;; CHECK-NEXT:   )
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT:  (unreachable)
+  ;; CHECK-NEXT: )
+  (func $A (result (ref $A))
+    (ref.cast_desc (ref $A)
+      (struct.new_default $A2
+        (struct.new_default $B2)
+      )
+      (struct.new_default $B)
+    )
+  )
+)