Get rid of SHAPE_T_OBJECT

Now the `heap_index` is stored as flags inside `shape_id_t`.

This saves having to rebuild the entire tree when moving from one
slot size to the other.

Author: byroot

ext/objspace/objspace_dump.c

@@ -817,9 +817,6 @@ shape_id_i(shape_id_t shape_id, void *data)
         dump_append(dc, ",\"edge_name\":");
         dump_append_id(dc, shape->edge_name);
 
-        break;
-      case SHAPE_T_OBJECT:
-        dump_append(dc, ", \"shape_type\":\"T_OBJECT\"");
         break;
       case SHAPE_OBJ_ID:
         dump_append(dc, ", \"shape_type\":\"OBJ_ID\"");

gc.c

@@ -379,19 +379,7 @@ rb_gc_set_shape(VALUE obj, uint32_t shape_id)
 uint32_t
 rb_gc_rebuild_shape(VALUE obj, size_t heap_id)
 {
-    shape_id_t orig_shape_id = rb_obj_shape_id(obj);
-    if (rb_shape_too_complex_p(orig_shape_id)) {
-        return (uint32_t)orig_shape_id;
-    }
-
-    shape_id_t initial_shape_id = rb_shape_root(heap_id);
-    shape_id_t new_shape_id = rb_shape_traverse_from_new_root(initial_shape_id, orig_shape_id);
-
-    if (new_shape_id == INVALID_SHAPE_ID) {
-         return 0;
-     }
-
-    return (uint32_t)new_shape_id;
+    return (uint32_t)rb_shape_transition_heap(obj, heap_id);
 }
 
 void rb_vm_update_references(void *ptr);
@@ -1923,6 +1911,7 @@ object_id0(VALUE obj)
     // rb_shape_object_id_shape may lock if the current shape has
     // multiple children.
     shape_id_t object_id_shape_id = rb_shape_transition_object_id(obj);
+    RUBY_ASSERT(rb_shape_has_object_id(object_id_shape_id));
 
     id = generate_next_object_id();
     rb_obj_field_set(obj, object_id_shape_id, id);

object.c

@@ -339,17 +339,16 @@ rb_obj_copy_ivar(VALUE dest, VALUE obj)
     shape_id_t dest_shape_id = src_shape_id;
     shape_id_t initial_shape_id = RBASIC_SHAPE_ID(dest);
 
-    if (RSHAPE(initial_shape_id)->heap_index != RSHAPE(src_shape_id)->heap_index || !rb_shape_canonical_p(src_shape_id)) {
-        RUBY_ASSERT(RSHAPE(initial_shape_id)->type == SHAPE_T_OBJECT);
+    RUBY_ASSERT(RSHAPE(initial_shape_id)->type == SHAPE_ROOT);
 
-        dest_shape_id = rb_shape_rebuild(initial_shape_id, src_shape_id);
-        if (UNLIKELY(rb_shape_too_complex_p(dest_shape_id))) {
-            st_table *table = rb_st_init_numtable_with_size(src_num_ivs);
-            rb_obj_copy_ivs_to_hash_table(obj, table);
-            rb_obj_init_too_complex(dest, table);
+    dest_shape_id = rb_shape_rebuild(initial_shape_id, src_shape_id);
 
-            return;
-        }
+    if (UNLIKELY(rb_shape_too_complex_p(dest_shape_id))) {
+        st_table *table = rb_st_init_numtable_with_size(src_num_ivs);
+        rb_obj_copy_ivs_to_hash_table(obj, table);
+        rb_obj_init_too_complex(dest, table);
+
+        return;
     }
 
     VALUE *src_buf = ROBJECT_FIELDS(obj);
@@ -365,7 +364,7 @@ rb_obj_copy_ivar(VALUE dest, VALUE obj)
     }
 
     rb_shape_copy_fields(dest, dest_buf, dest_shape_id, obj, src_buf, src_shape_id);
-    rb_obj_set_shape_id(dest, dest_shape_id);
+    RBASIC_SET_SHAPE_ID(dest, dest_shape_id);
 }
 
 static void

shape.c

@@ -304,14 +304,7 @@ shape_tree_mark(void *data)
     rb_shape_t *end = RSHAPE(GET_SHAPE_TREE()->next_shape_id - 1);
     while (cursor < end) {
         if (cursor->edges && !SINGLE_CHILD_P(cursor->edges)) {
-            // FIXME: GC compaction may call `rb_shape_traverse_from_new_root`
-            // to migrate objects from one object slot to another.
-            // Because of this if we don't pin `cursor->edges` it might be turned
-            // into a T_MOVED during GC.
-            // We'd need to eliminate `SHAPE_T_OBJECT` so that GC never need to lookup
-            // shapes this way.
-            // rb_gc_mark_movable(cursor->edges);
-            rb_gc_mark(cursor->edges);
+            rb_gc_mark_movable(cursor->edges);
         }
         cursor++;
     }
@@ -368,7 +361,16 @@ shape_id(rb_shape_t *shape, shape_id_t previous_shape_id)
         return INVALID_SHAPE_ID;
     }
     shape_id_t raw_id = (shape_id_t)(shape - GET_SHAPE_TREE()->shape_list);
-    return raw_id | (previous_shape_id & SHAPE_ID_FLAGS_MASK);
+    shape_id_t new_id = raw_id | (previous_shape_id & SHAPE_ID_FLAGS_MASK);
+
+    attr_index_t embed_capa = RSHAPE_EMBED_CAPA(previous_shape_id);
+    if (embed_capa > shape->next_field_index) {
+        new_id |= SHAPE_ID_FL_EMBEDDED;
+    }
+    else {
+        new_id &= ~SHAPE_ID_FL_EMBEDDED;
+    }
+    return new_id;
 }
 
 #if RUBY_DEBUG
@@ -455,7 +457,6 @@ rb_shape_alloc(ID edge_name, rb_shape_t *parent, enum shape_type type)
 {
     rb_shape_t *shape = rb_shape_alloc_with_parent_id(edge_name, raw_shape_id(parent));
     shape->type = (uint8_t)type;
-    shape->heap_index = parent->heap_index;
     shape->capacity = parent->capacity;
     shape->edges = 0;
     return shape;
@@ -515,7 +516,6 @@ rb_shape_alloc_new_child(ID id, rb_shape_t *shape, enum shape_type shape_type)
         }
         break;
       case SHAPE_ROOT:
-      case SHAPE_T_OBJECT:
         rb_bug("Unreachable");
         break;
     }
@@ -743,7 +743,7 @@ rb_shape_transition_frozen(VALUE obj)
 {
     RUBY_ASSERT(RB_OBJ_FROZEN(obj));
 
-    shape_id_t shape_id = rb_obj_shape_id(obj);
+    shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
     return shape_id | SHAPE_ID_FL_FROZEN;
 }
 
@@ -753,6 +753,12 @@ rb_shape_transition_complex(VALUE obj)
     return transition_frozen(RBASIC_SHAPE_ID(obj));
 }
 
+shape_id_t
+rb_shape_transition_heap(VALUE obj, size_t heap_index)
+{
+    return (RBASIC_SHAPE_ID(obj) & (~SHAPE_ID_HEAP_INDEX_MASK)) | rb_shape_root(heap_index);
+}
+
 shape_id_t
 rb_shape_transition_object_id(VALUE obj)
 {
@@ -822,7 +828,6 @@ shape_get_iv_index(rb_shape_t *shape, ID id, attr_index_t *value)
                 *value = shape->next_field_index - 1;
                 return true;
               case SHAPE_ROOT:
-              case SHAPE_T_OBJECT:
                 return false;
               case SHAPE_OBJ_ID:
                 rb_bug("Ivar should not exist on transition");
@@ -1019,61 +1024,6 @@ rb_shape_id_offset(void)
     return sizeof(uintptr_t) - SHAPE_ID_NUM_BITS / sizeof(uintptr_t);
 }
 
-static rb_shape_t *
-shape_traverse_from_new_root(rb_shape_t *initial_shape, rb_shape_t *dest_shape)
-{
-    RUBY_ASSERT(initial_shape->type == SHAPE_T_OBJECT);
-    rb_shape_t *next_shape = initial_shape;
-
-    if (dest_shape->type != initial_shape->type) {
-        next_shape = shape_traverse_from_new_root(initial_shape, RSHAPE(dest_shape->parent_id));
-        if (!next_shape) {
-            return NULL;
-        }
-    }
-
-    switch ((enum shape_type)dest_shape->type) {
-      case SHAPE_IVAR:
-      case SHAPE_OBJ_ID:
-        if (!next_shape->edges) {
-            return NULL;
-        }
-
-        VALUE lookup_result;
-        if (SINGLE_CHILD_P(next_shape->edges)) {
-            rb_shape_t *child = SINGLE_CHILD(next_shape->edges);
-            if (child->edge_name == dest_shape->edge_name) {
-                return child;
-            }
-            else {
-                return NULL;
-            }
-        }
-        else {
-            if (rb_managed_id_table_lookup(next_shape->edges, dest_shape->edge_name, &lookup_result)) {
-                next_shape = (rb_shape_t *)lookup_result;
-            }
-            else {
-                return NULL;
-            }
-        }
-        break;
-      case SHAPE_ROOT:
-      case SHAPE_T_OBJECT:
-        break;
-    }
-
-    return next_shape;
-}
-
-shape_id_t
-rb_shape_traverse_from_new_root(shape_id_t initial_shape_id, shape_id_t dest_shape_id)
-{
-    rb_shape_t *initial_shape = RSHAPE(initial_shape_id);
-    rb_shape_t *dest_shape = RSHAPE(dest_shape_id);
-    return shape_id(shape_traverse_from_new_root(initial_shape, dest_shape), dest_shape_id);
-}
-
 // Rebuild a similar shape with the same ivars but starting from
 // a different SHAPE_T_OBJECT, and don't cary over non-canonical transitions
 // such as SHAPE_OBJ_ID.
@@ -1082,7 +1032,7 @@ shape_rebuild(rb_shape_t *initial_shape, rb_shape_t *dest_shape)
 {
     rb_shape_t *midway_shape;
 
-    RUBY_ASSERT(initial_shape->type == SHAPE_T_OBJECT || initial_shape->type == SHAPE_ROOT);
+    RUBY_ASSERT(initial_shape->type == SHAPE_ROOT);
 
     if (dest_shape->type != initial_shape->type) {
         midway_shape = shape_rebuild(initial_shape, RSHAPE(dest_shape->parent_id));
@@ -1100,7 +1050,6 @@ shape_rebuild(rb_shape_t *initial_shape, rb_shape_t *dest_shape)
         break;
       case SHAPE_OBJ_ID:
       case SHAPE_ROOT:
-      case SHAPE_T_OBJECT:
         break;
     }
 
@@ -1113,7 +1062,14 @@ rb_shape_rebuild(shape_id_t initial_shape_id, shape_id_t dest_shape_id)
     RUBY_ASSERT(!rb_shape_too_complex_p(initial_shape_id));
     RUBY_ASSERT(!rb_shape_too_complex_p(dest_shape_id));
 
-    return raw_shape_id(shape_rebuild(RSHAPE(initial_shape_id), RSHAPE(dest_shape_id)));
+    if (rb_shape_has_object_id(dest_shape_id)) {
+        // The tree contains a SHAPE_OBJ_ID. We need to rebuild a need tree without it
+        return shape_id(shape_rebuild(RSHAPE(initial_shape_id), RSHAPE(dest_shape_id)), initial_shape_id & ~SHAPE_ID_FL_NON_CANONICAL_MASK);
+    }
+    else {
+        // In the simple case we keep the tree from `dest_shape_id` and the flags from `initial_shape_id`.
+        return (dest_shape_id & ~SHAPE_ID_FLAGS_MASK) | (initial_shape_id & SHAPE_ID_FLAGS_MASK);
+    }
 }
 
 void
@@ -1122,6 +1078,10 @@ rb_shape_copy_fields(VALUE dest, VALUE *dest_buf, shape_id_t dest_shape_id, VALU
     rb_shape_t *dest_shape = RSHAPE(dest_shape_id);
     rb_shape_t *src_shape = RSHAPE(src_shape_id);
 
+    if (!dest_shape->next_field_index) {
+        return;
+    }
+
     if (src_shape->next_field_index == dest_shape->next_field_index) {
         // Happy path, we can just memcpy the ivptr content
         MEMCPY(dest_buf, src_buf, VALUE, dest_shape->next_field_index);
@@ -1135,6 +1095,9 @@ rb_shape_copy_fields(VALUE dest, VALUE *dest_buf, shape_id_t dest_shape_id, VALU
         while (src_shape->parent_id != INVALID_SHAPE_ID) {
             if (src_shape->type == SHAPE_IVAR) {
                 while (dest_shape->edge_name != src_shape->edge_name) {
+                    if (UNLIKELY(dest_shape->parent_id == INVALID_SHAPE_ID)) {
+                        rb_bug("Lost field %s", rb_id2name(src_shape->edge_name));
+                    }
                     dest_shape = RSHAPE(dest_shape->parent_id);
                 }
 
@@ -1212,6 +1175,16 @@ rb_shape_verify_consistency(VALUE obj, shape_id_t shape_id)
         }
     }
 
+    if (RB_TYPE_P(obj, T_OBJECT)) {
+        size_t slot_size = rb_gc_obj_slot_size(obj);
+        size_t real_embed_capa = (slot_size - sizeof(struct RBasic)) / sizeof(VALUE);
+        size_t shape_embed_capa = RSHAPE_EMBED_CAPA(shape_id);
+
+        if (real_embed_capa != shape_embed_capa) {
+            rb_bug("Expected real_embed_capa=%lu to match shape_embed_capa=%lu\n", real_embed_capa, shape_embed_capa);
+        }
+    }
+
     return true;
 }
 #endif
@@ -1259,13 +1232,15 @@ shape_id_t_to_rb_cShape(shape_id_t shape_id)
 {
     VALUE rb_cShape = rb_const_get(rb_cRubyVM, rb_intern("Shape"));
     rb_shape_t *shape = RSHAPE(shape_id);
+    size_t heap_index = rb_shape_heap_index(shape_id);
 
     VALUE obj = rb_struct_new(rb_cShape,
             INT2NUM(shape_id),
+            INT2NUM(shape_id & SHAPE_ID_OFFSET_MASK),
             INT2NUM(shape->parent_id),
             rb_shape_edge_name(shape),
             INT2NUM(shape->next_field_index),
-            INT2NUM(shape->heap_index),
+            heap_index ? SIZET2NUM(heap_index - 1) : Qnil,
             INT2NUM(shape->type),
             INT2NUM(shape->capacity));
     rb_obj_freeze(obj);
@@ -1473,28 +1448,15 @@ Init_default_shapes(void)
     rb_shape_t *root = rb_shape_alloc_with_parent_id(0, INVALID_SHAPE_ID);
     root->capacity = 0;
     root->type = SHAPE_ROOT;
-    root->heap_index = 0;
     GET_SHAPE_TREE()->root_shape = root;
     RUBY_ASSERT(raw_shape_id(GET_SHAPE_TREE()->root_shape) == ROOT_SHAPE_ID);
 
     rb_shape_t *root_with_obj_id = rb_shape_alloc_with_parent_id(0, ROOT_SHAPE_ID);
     root_with_obj_id->type = SHAPE_OBJ_ID;
     root_with_obj_id->edge_name = ruby_internal_object_id;
     root_with_obj_id->next_field_index++;
-    root_with_obj_id->heap_index = 0;
     RUBY_ASSERT(raw_shape_id(root_with_obj_id) == ROOT_SHAPE_WITH_OBJ_ID);
-
-    // Make shapes for T_OBJECT
-    size_t *sizes = rb_gc_heap_sizes();
-    for (int i = 0; sizes[i] > 0; i++) {
-        rb_shape_t *t_object_shape = rb_shape_alloc_with_parent_id(0, INVALID_SHAPE_ID);
-        t_object_shape->type = SHAPE_T_OBJECT;
-        t_object_shape->heap_index = i;
-        t_object_shape->capacity = (uint32_t)((sizes[i] - offsetof(struct RObject, as.ary)) / sizeof(VALUE));
-        t_object_shape->edges = rb_managed_id_table_new(256);
-        t_object_shape->ancestor_index = LEAF;
-        RUBY_ASSERT(t_object_shape == RSHAPE(rb_shape_root(i)));
-    }
+    RUBY_ASSERT(RSHAPE(ROOT_TOO_COMPLEX_WITH_OBJ_ID)->type == SHAPE_OBJ_ID);
 }
 
 void
@@ -1511,6 +1473,7 @@ Init_shape(void)
      * :nodoc: */
     VALUE rb_cShape = rb_struct_define_under(rb_cRubyVM, "Shape",
             "id",
+            "raw_id",
             "parent_id",
             "edge_name",
             "next_field_index",
@@ -1528,11 +1491,9 @@ Init_shape(void)
 
     rb_define_const(rb_cShape, "SHAPE_ROOT", INT2NUM(SHAPE_ROOT));
     rb_define_const(rb_cShape, "SHAPE_IVAR", INT2NUM(SHAPE_IVAR));
-    rb_define_const(rb_cShape, "SHAPE_T_OBJECT", INT2NUM(SHAPE_T_OBJECT));
     rb_define_const(rb_cShape, "SHAPE_ID_NUM_BITS", INT2NUM(SHAPE_ID_NUM_BITS));
     rb_define_const(rb_cShape, "SHAPE_FLAG_SHIFT", INT2NUM(SHAPE_FLAG_SHIFT));
     rb_define_const(rb_cShape, "SPECIAL_CONST_SHAPE_ID", INT2NUM(SPECIAL_CONST_SHAPE_ID));
-    rb_define_const(rb_cShape, "FIRST_T_OBJECT_SHAPE_ID", INT2NUM(FIRST_T_OBJECT_SHAPE_ID));
     rb_define_const(rb_cShape, "SHAPE_MAX_VARIATIONS", INT2NUM(SHAPE_MAX_VARIATIONS));
     rb_define_const(rb_cShape, "SIZEOF_RB_SHAPE_T", INT2NUM(sizeof(rb_shape_t)));
     rb_define_const(rb_cShape, "SIZEOF_REDBLACK_NODE_T", INT2NUM(sizeof(redblack_node_t)));