Get rid of TOO_COMPLEX shape type

Complex shapes still exist, but it's now a `shape_id` bit flag
rather than a materialized shape transition.

This allows to check for complex shape without dereferencing
the shape.

Author: byroot

gc.c

@@ -1897,7 +1897,8 @@ object_id0(VALUE obj)
 {
     VALUE id = Qfalse;
 
-    if (rb_shape_has_object_id(RBASIC_SHAPE_ID(obj))) {
+    shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
+    if (rb_shape_has_object_id(shape_id)) {
         shape_id_t object_id_shape_id = rb_shape_transition_object_id(obj);
         id = rb_obj_field_get(obj, object_id_shape_id);
         RUBY_ASSERT(id, "object_id missing");

shape.c

@@ -461,7 +461,6 @@ rb_shape_alloc(ID edge_name, rb_shape_t *parent, enum shape_type type)
 {
     rb_shape_t *shape = rb_shape_alloc_with_parent_id(edge_name, raw_shape_id(parent));
     shape->type = (uint8_t)type;
-    shape->flags = parent->flags;
     shape->heap_index = parent->heap_index;
     shape->capacity = parent->capacity;
     shape->edges = 0;
@@ -510,8 +509,6 @@ rb_shape_alloc_new_child(ID id, rb_shape_t *shape, enum shape_type shape_type)
 
     switch (shape_type) {
       case SHAPE_OBJ_ID:
-        new_shape->flags |= SHAPE_FL_HAS_OBJECT_ID;
-        // fallthrough
       case SHAPE_IVAR:
         if (UNLIKELY(shape->next_field_index >= shape->capacity)) {
             RUBY_ASSERT(shape->next_field_index == shape->capacity);
@@ -753,18 +750,6 @@ rb_shape_transition_complex(VALUE obj)
     return ROOT_TOO_COMPLEX_SHAPE_ID | (original_shape_id & SHAPE_ID_FLAGS_MASK);
 }
 
-static inline bool
-shape_has_object_id(rb_shape_t *shape)
-{
-    return shape->flags & SHAPE_FL_HAS_OBJECT_ID;
-}
-
-bool
-rb_shape_has_object_id(shape_id_t shape_id)
-{
-    return shape_has_object_id(RSHAPE(shape_id));
-}
-
 shape_id_t
 rb_shape_transition_object_id(VALUE obj)
 {
@@ -773,7 +758,11 @@ rb_shape_transition_object_id(VALUE obj)
     rb_shape_t* shape = RSHAPE(original_shape_id);
     RUBY_ASSERT(shape);
 
-    if (shape->flags & SHAPE_FL_HAS_OBJECT_ID) {
+    if (rb_shape_has_object_id(original_shape_id)) {
+        if (rb_shape_too_complex_p(original_shape_id)) {
+            return ROOT_SHAPE_WITH_OBJ_ID | SHAPE_ID_FL_HAS_OBJECT_ID | SHAPE_ID_FL_TOO_COMPLEX;
+        }
+
         while (shape->type != SHAPE_OBJ_ID) {
             shape = RSHAPE(shape->parent_id);
         }
@@ -783,7 +772,7 @@ rb_shape_transition_object_id(VALUE obj)
         shape = get_next_shape_internal(shape, ruby_internal_object_id, SHAPE_OBJ_ID, &dont_care, true);
     }
     RUBY_ASSERT(shape);
-    return shape_id(shape, original_shape_id);
+    return shape_id(shape, original_shape_id) | SHAPE_ID_FL_HAS_OBJECT_ID;
 }
 
 /*
@@ -1203,8 +1192,7 @@ static VALUE
 shape_has_object_id_p(VALUE self)
 {
     shape_id_t shape_id = NUM2INT(rb_struct_getmember(self, rb_intern("id")));
-    rb_shape_t *shape = RSHAPE(shape_id);
-    return RBOOL(shape_has_object_id(shape));
+    return RBOOL(rb_shape_has_object_id(shape_id));
 }
 
 static VALUE
@@ -1441,6 +1429,11 @@ Init_default_shapes(void)
     GET_SHAPE_TREE()->root_shape = root;
     RUBY_ASSERT(raw_shape_id(GET_SHAPE_TREE()->root_shape) == ROOT_SHAPE_ID);
 
+    rb_shape_t *root_with_obj_id = rb_shape_alloc_with_parent_id(0, ROOT_SHAPE_ID);
+    root_with_obj_id->type = SHAPE_OBJ_ID;
+    root_with_obj_id->heap_index = 0;
+    RUBY_ASSERT(raw_shape_id(root_with_obj_id) == ROOT_SHAPE_WITH_OBJ_ID);
+
     // Make shapes for T_OBJECT
     size_t *sizes = rb_gc_heap_sizes();
     for (int i = 0; sizes[i] > 0; i++) {

shape.h

@@ -14,7 +14,9 @@ STATIC_ASSERT(shape_id_num_bits, SHAPE_ID_NUM_BITS == sizeof(shape_id_t) * CHAR_
 #define SHAPE_ID_OFFSET_MASK (SHAPE_BUFFER_SIZE - 1)
 #define SHAPE_ID_FLAGS_MASK (shape_id_t)(((1 << (SHAPE_ID_NUM_BITS - SHAPE_ID_OFFSET_NUM_BITS)) - 1) << SHAPE_ID_OFFSET_NUM_BITS)
 #define SHAPE_ID_FL_FROZEN (SHAPE_FL_FROZEN << SHAPE_ID_OFFSET_NUM_BITS)
+#define SHAPE_ID_FL_HAS_OBJECT_ID (SHAPE_FL_HAS_OBJECT_ID << SHAPE_ID_OFFSET_NUM_BITS)
 #define SHAPE_ID_FL_TOO_COMPLEX (SHAPE_FL_TOO_COMPLEX << SHAPE_ID_OFFSET_NUM_BITS)
+#define SHAPE_ID_FL_NON_CANONICAL_MASK (SHAPE_FL_NON_CANONICAL_MASK << SHAPE_ID_OFFSET_NUM_BITS)
 #define SHAPE_ID_READ_ONLY_MASK (~SHAPE_ID_FL_FROZEN)
 
 typedef uint32_t redblack_id_t;
@@ -29,9 +31,10 @@ typedef uint32_t redblack_id_t;
 #define ATTR_INDEX_NOT_SET ((attr_index_t)-1)
 
 #define ROOT_SHAPE_ID               0x0
+#define ROOT_SHAPE_WITH_OBJ_ID      0x1
 #define ROOT_TOO_COMPLEX_SHAPE_ID   (ROOT_SHAPE_ID | SHAPE_ID_FL_TOO_COMPLEX)
 #define SPECIAL_CONST_SHAPE_ID      (ROOT_SHAPE_ID | SHAPE_ID_FL_FROZEN)
-#define FIRST_T_OBJECT_SHAPE_ID     0x1
+#define FIRST_T_OBJECT_SHAPE_ID     0x2
 
 extern ID ruby_internal_object_id;
 
@@ -46,7 +49,6 @@ struct rb_shape {
     attr_index_t capacity; // Total capacity of the object with this shape
     uint8_t type;
     uint8_t heap_index;
-    uint8_t flags;
 };
 
 typedef struct rb_shape rb_shape_t;
@@ -142,7 +144,6 @@ RUBY_FUNC_EXPORTED shape_id_t rb_obj_shape_id(VALUE obj);
 shape_id_t rb_shape_get_next_iv_shape(shape_id_t shape_id, ID id);
 bool rb_shape_get_iv_index(shape_id_t shape_id, ID id, attr_index_t *value);
 bool rb_shape_get_iv_index_with_hint(shape_id_t shape_id, ID id, attr_index_t *value, shape_id_t *shape_id_hint);
-bool rb_shape_has_object_id(shape_id_t shape_id);
 
 shape_id_t rb_shape_transition_frozen(VALUE obj);
 shape_id_t rb_shape_transition_complex(VALUE obj);
@@ -169,10 +170,16 @@ rb_shape_obj_too_complex_p(VALUE obj)
     return !RB_SPECIAL_CONST_P(obj) && rb_shape_too_complex_p(RBASIC_SHAPE_ID(obj));
 }
 
+static inline bool
+rb_shape_has_object_id(shape_id_t shape_id)
+{
+    return shape_id & SHAPE_ID_FL_HAS_OBJECT_ID;
+}
+
 static inline bool
 rb_shape_canonical_p(shape_id_t shape_id)
 {
-    return !(shape_id & SHAPE_ID_FLAGS_MASK) && !RSHAPE(shape_id)->flags;
+    return !(shape_id & SHAPE_ID_FL_NON_CANONICAL_MASK);
 }
 
 static inline shape_id_t