Use shape bits to store capacity of object

This commit uses 7 shape bits to store the capacity of the object instead
of the heap ID. This allows for slot sizes with up to capacity of 127.
This removes the abstraction leak of slot sizes from the GC, which allows
more flexibility in the GC.

This implementation should currently make no difference for the default
GC as it will end up creating the same root shapes. We can see that there
is basically no change in benchmarks:

--------------  ------------  ------------  ------------  ------------  --------------  -------------  -----------------
bench            master (ms)     RSS (MiB)   branch (ms)     RSS (MiB)  branch 1st itr  master/branch  RSS master/branch
activerecord     99.6 ± 3.9%   75.5 ± 0.3%   98.9 ± 2.4%   74.0 ± 1.3%           0.919          1.007              1.021
chunky-png      328.2 ± 0.6%   83.0 ± 3.9%  336.4 ± 0.7%   87.0 ± 2.2%           0.979          0.976              0.954
erubi-rails     440.4 ± 1.7%  138.1 ± 0.3%  431.0 ± 0.5%  135.2 ± 0.0%           0.992          1.022              1.022
hexapdf         855.6 ± 0.7%  577.0 ± 0.1%  856.7 ± 0.9%  649.0 ± 2.6%           0.993          0.999              0.889
liquid-c         21.6 ± 8.2%  69.6 ± 14.7%   23.2 ± 8.4%  64.6 ± 15.4%           0.993          0.930              1.077
liquid-compile   19.9 ± 7.9%   47.0 ± 5.7%   20.2 ± 8.7%   46.9 ± 5.1%           0.925          0.987              1.001
liquid-render    54.0 ± 2.7%   55.3 ± 8.6%   55.2 ± 2.7%   55.9 ± 9.0%           0.977          0.978              0.990
lobsters        351.6 ± 0.7%  336.8 ± 0.2%  357.0 ± 0.6%  335.4 ± 0.1%           0.997          0.985              1.004
mail             47.5 ± 3.1%   75.6 ± 2.7%   46.8 ± 1.3%   72.3 ± 0.5%           0.974          1.016              1.045
psych-load      829.1 ± 1.2%   55.0 ± 0.4%  832.9 ± 0.5%   55.3 ± 0.7%           1.006          0.995              0.995
railsbench      713.8 ± 0.4%  134.6 ± 1.1%  717.9 ± 1.0%  138.2 ± 1.2%           0.992          0.994              0.974
rubocop          73.1 ± 6.0%  106.7 ± 1.7%   73.2 ± 6.1%  106.4 ± 1.4%           0.997          0.998              1.003
ruby-lsp         69.0 ± 1.2%   78.3 ± 0.2%   67.8 ± 2.3%   74.2 ± 0.0%           1.043          1.017              1.055
sequel           21.7 ± 6.8%   56.1 ± 1.6%   21.0 ± 4.4%   55.9 ± 1.1%           0.873          1.032              1.003
shipit          608.9 ± 1.4%  160.2 ± 0.5%  619.2 ± 1.4%  157.3 ± 0.2%           1.012          0.983              1.018
--------------  ------------  ------------  ------------  ------------  --------------  -------------  -----------------

Author: peterzhu2118

gc.c

@@ -375,11 +375,11 @@ rb_gc_shutdown_call_finalizer_p(VALUE obj)
 }
 
 void
-rb_gc_obj_changed_pool(VALUE obj, size_t heap_id)
+rb_gc_obj_changed_slot_size(VALUE obj, size_t slot_size)
 {
     RUBY_ASSERT(RB_TYPE_P(obj, T_OBJECT));
 
-    RBASIC_SET_SHAPE_ID(obj, rb_obj_shape_transition_heap(obj, heap_id));
+    RBASIC_SET_SHAPE_ID_WITH_CAPACITY(obj, rb_obj_shape_transition_capacity(obj, rb_shape_capacity_for_slot_size(slot_size)));
 }
 
 void rb_vm_update_references(void *ptr);
@@ -604,7 +604,6 @@ typedef struct gc_function_map {
     void *(*ractor_cache_alloc)(void *objspace_ptr, void *ractor);
     void (*set_params)(void *objspace_ptr);
     void (*init)(void);
-    size_t *(*heap_sizes)(void *objspace_ptr);
     // Shutdown
     void (*shutdown_free_objects)(void *objspace_ptr);
     void (*objspace_free)(void *objspace_ptr);
@@ -622,9 +621,9 @@ typedef struct gc_function_map {
     VALUE (*stress_get)(void *objspace_ptr);
     struct rb_gc_vm_context *(*get_vm_context)(void *objspace_ptr);
     // Object allocation
-    VALUE (*new_obj)(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size);
+    VALUE (*new_obj)(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size, size_t *actual_alloc_size);
     size_t (*obj_slot_size)(VALUE obj);
-    size_t (*heap_id_for_size)(void *objspace_ptr, size_t size);
+    size_t (*size_slot_size)(void *objspace_ptr, size_t size);
     bool (*size_allocatable_p)(size_t size);
     // Malloc
     void *(*malloc)(void *objspace_ptr, size_t size, bool gc_allowed);
@@ -783,7 +782,6 @@ ruby_modular_gc_init(void)
     load_modular_gc_func(ractor_cache_alloc);
     load_modular_gc_func(set_params);
     load_modular_gc_func(init);
-    load_modular_gc_func(heap_sizes);
     // Shutdown
     load_modular_gc_func(shutdown_free_objects);
     load_modular_gc_func(objspace_free);
@@ -803,7 +801,7 @@ ruby_modular_gc_init(void)
     // Object allocation
     load_modular_gc_func(new_obj);
     load_modular_gc_func(obj_slot_size);
-    load_modular_gc_func(heap_id_for_size);
+    load_modular_gc_func(size_slot_size);
     load_modular_gc_func(size_allocatable_p);
     // Malloc
     load_modular_gc_func(malloc);
@@ -871,7 +869,6 @@ ruby_modular_gc_init(void)
 # define rb_gc_impl_ractor_cache_alloc rb_gc_functions.ractor_cache_alloc
 # define rb_gc_impl_set_params rb_gc_functions.set_params
 # define rb_gc_impl_init rb_gc_functions.init
-# define rb_gc_impl_heap_sizes rb_gc_functions.heap_sizes
 // Shutdown
 # define rb_gc_impl_shutdown_free_objects rb_gc_functions.shutdown_free_objects
 # define rb_gc_impl_objspace_free rb_gc_functions.objspace_free
@@ -891,7 +888,7 @@ ruby_modular_gc_init(void)
 // Object allocation
 # define rb_gc_impl_new_obj rb_gc_functions.new_obj
 # define rb_gc_impl_obj_slot_size rb_gc_functions.obj_slot_size
-# define rb_gc_impl_heap_id_for_size rb_gc_functions.heap_id_for_size
+# define rb_gc_impl_size_slot_size rb_gc_functions.size_slot_size
 # define rb_gc_impl_size_allocatable_p rb_gc_functions.size_allocatable_p
 // Malloc
 # define rb_gc_impl_malloc rb_gc_functions.malloc
@@ -1033,7 +1030,11 @@ rb_newobj(rb_execution_context_t *ec, VALUE klass, VALUE flags, shape_id_t shape
 {
     GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
     rb_ractor_t *cr = rb_ec_ractor_ptr(ec);
-    VALUE obj = rb_gc_impl_new_obj(rb_gc_get_objspace(), cr->newobj_cache, klass, flags, wb_protected, size);
+    size_t actual_alloc_size;
+    VALUE obj = rb_gc_impl_new_obj(rb_gc_get_objspace(), cr->newobj_cache, klass, flags, wb_protected, size, &actual_alloc_size);
+
+    GC_ASSERT(actual_alloc_size >= size);
+    shape_id = (shape_id & ~SHAPE_ID_CAPACITY_MASK) | rb_shape_id_with_capacity(rb_shape_capacity_for_slot_size(actual_alloc_size));
 
 #if RACTOR_CHECK_MODE
     void rb_ractor_setup_belonging(VALUE obj);
@@ -1092,7 +1093,7 @@ rb_newobj_of(VALUE klass, VALUE flags, size_t size)
 static
 VALUE class_allocate_complex_instance(VALUE klass, uint32_t capacity)
 {
-    shape_id_t initial_shape_id = rb_shape_id_with_robject_layout(rb_shape_root(rb_gc_heap_id_for_size(sizeof(struct RObject))));
+    shape_id_t initial_shape_id = rb_shape_id_with_robject_layout(0);
     VALUE obj = rb_newobj_of_with_shape(klass, T_OBJECT, initial_shape_id, sizeof(struct RObject));
     rb_obj_init_complex(obj, rb_st_init_numtable_with_size(capacity));
     return obj;
@@ -1105,8 +1106,8 @@ rb_class_allocate_instance(VALUE klass)
     VALUE obj;
 
     // Directly start as COMPLEX if we know we're over the limit.
-    RUBY_ASSERT(rb_shape_tree.max_capacity > 0);
-    if (RB_UNLIKELY(index_tbl_num_entries > rb_shape_tree.max_capacity)) {
+    RUBY_ASSERT(SHAPE_MAX_CAPACITY > 0);
+    if (RB_UNLIKELY(index_tbl_num_entries > SHAPE_MAX_CAPACITY)) {
         obj = class_allocate_complex_instance(klass, index_tbl_num_entries);
     }
     else {
@@ -1117,7 +1118,7 @@ rb_class_allocate_instance(VALUE klass)
 
         // There might be a NEWOBJ tracepoint callback, and it may set fields.
         // So the shape must be passed to `NEWOBJ_OF`.
-        obj = rb_newobj_of_with_shape(klass, T_OBJECT, rb_shape_id_with_robject_layout(rb_shape_root(rb_gc_heap_id_for_size(size))), size);
+        obj = rb_newobj_of_with_shape(klass, T_OBJECT, rb_shape_id_with_robject_layout(0), size);
 
         #if RUBY_DEBUG
             VALUE *ptr = ROBJECT_FIELDS(obj);
@@ -2187,7 +2188,6 @@ object_id0(VALUE obj)
     id = generate_next_object_id();
     rb_obj_field_set(obj, object_id_shape_id, 0, id);
 
-    RUBY_ASSERT(RBASIC_SHAPE_ID(obj) == object_id_shape_id);
     RUBY_ASSERT(rb_obj_shape_has_id(obj));
 
     if (RB_UNLIKELY(id2ref_tbl)) {
@@ -3989,9 +3989,9 @@ rb_gc_prepare_heap(void)
 }
 
 size_t
-rb_gc_heap_id_for_size(size_t size)
+rb_gc_size_slot_size(size_t size)
 {
-    return rb_gc_impl_heap_id_for_size(rb_gc_get_objspace(), size);
+    return rb_gc_impl_size_slot_size(rb_gc_get_objspace(), size);
 }
 
 bool
@@ -4704,12 +4704,6 @@ rb_gc_initial_stress_set(VALUE flag)
     initial_stress = flag;
 }
 
-size_t *
-rb_gc_heap_sizes(void)
-{
-    return rb_gc_impl_heap_sizes(rb_gc_get_objspace());
-}
-
 VALUE
 rb_gc_enable(void)
 {

gc/default/default.c

@@ -2606,24 +2606,9 @@ heap_idx_for_size(size_t size)
 }
 
 size_t
-rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size)
+rb_gc_impl_size_slot_size(void *objspace_ptr, size_t size)
 {
-    return heap_idx_for_size(size);
-}
-
-
-static size_t heap_sizes[HEAP_COUNT + 1] = { 0 };
-
-size_t *
-rb_gc_impl_heap_sizes(void *objspace_ptr)
-{
-    if (heap_sizes[0] == 0) {
-        for (unsigned char i = 0; i < HEAP_COUNT; i++) {
-            heap_sizes[i] = heap_slot_size(i);
-        }
-    }
-
-    return heap_sizes;
+    return heap_slot_size((unsigned char)heap_idx_for_size(size));
 }
 
 NOINLINE(static VALUE newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx, bool vm_locked));
@@ -2736,7 +2721,7 @@ newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace
 }
 
 VALUE
-rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size)
+rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size, size_t *actual_alloc_size)
 {
     VALUE obj;
     rb_objspace_t *objspace = objspace_ptr;
@@ -2751,6 +2736,7 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags
     }
 
     size_t heap_idx = heap_idx_for_size(alloc_size);
+    *actual_alloc_size = heap_slot_size((unsigned char)heap_idx);
 
     rb_ractor_newobj_cache_t *cache = (rb_ractor_newobj_cache_t *)cache_ptr;
 
@@ -7479,7 +7465,7 @@ gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, struct heap_page *src_pa
     memcpy((void *)dest, (void *)src, MIN(src_slot_size, slot_size));
 
     if (src_slot_size != slot_size && RB_TYPE_P(src, T_OBJECT)) {
-        rb_gc_obj_changed_pool(dest, dest_page->heap - heaps);
+        rb_gc_obj_changed_slot_size(dest, slot_size - RVALUE_OVERHEAD);
     }
 
     if (RVALUE_OVERHEAD > 0) {

gc/gc.h

@@ -110,7 +110,7 @@ MODULAR_GC_FN void rb_gc_mark_roots(void *objspace, const char **categoryp);
 MODULAR_GC_FN void rb_gc_ractor_newobj_cache_foreach(void (*func)(void *cache, void *data), void *data);
 MODULAR_GC_FN bool rb_gc_multi_ractor_p(void);
 MODULAR_GC_FN bool rb_gc_shutdown_call_finalizer_p(VALUE obj);
-MODULAR_GC_FN void rb_gc_obj_changed_pool(VALUE obj, size_t heap_id);
+MODULAR_GC_FN void rb_gc_obj_changed_slot_size(VALUE obj, size_t slot_size);
 MODULAR_GC_FN void rb_gc_prepare_heap_process_object(VALUE obj);
 MODULAR_GC_FN bool rb_memerror_reentered(void);
 MODULAR_GC_FN bool rb_obj_id_p(VALUE);

gc/gc_impl.h

@@ -38,7 +38,6 @@ GC_IMPL_FN void rb_gc_impl_objspace_init(void *objspace_ptr);
 GC_IMPL_FN void *rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor);
 GC_IMPL_FN void rb_gc_impl_set_params(void *objspace_ptr);
 GC_IMPL_FN void rb_gc_impl_init(void);
-GC_IMPL_FN size_t *rb_gc_impl_heap_sizes(void *objspace_ptr);
 // Shutdown
 GC_IMPL_FN void rb_gc_impl_shutdown_free_objects(void *objspace_ptr);
 GC_IMPL_FN void rb_gc_impl_objspace_free(void *objspace_ptr);
@@ -56,9 +55,9 @@ GC_IMPL_FN VALUE rb_gc_impl_config_get(void *objspace_ptr);
 GC_IMPL_FN void rb_gc_impl_config_set(void *objspace_ptr, VALUE hash);
 GC_IMPL_FN struct rb_gc_vm_context *rb_gc_impl_get_vm_context(void *objspace_ptr);
 // Object allocation
-GC_IMPL_FN VALUE rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size);
+GC_IMPL_FN VALUE rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size, size_t *actual_alloc_size);
 GC_IMPL_FN size_t rb_gc_impl_obj_slot_size(VALUE obj);
-GC_IMPL_FN size_t rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size);
+GC_IMPL_FN size_t rb_gc_impl_size_slot_size(void *objspace_ptr, size_t size);
 GC_IMPL_FN bool rb_gc_impl_size_allocatable_p(size_t size);
 // Malloc
 /*

gc/mmtk/mmtk.c

@@ -687,12 +687,6 @@ rb_gc_impl_init(void)
     rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1);
 }
 
-size_t *
-rb_gc_impl_heap_sizes(void *objspace_ptr)
-{
-    return heap_sizes;
-}
-
 int
 rb_mmtk_obj_free_iter_wrapper(VALUE obj, void *data)
 {
@@ -902,7 +896,7 @@ mmtk_post_alloc_fast_immix(struct objspace *objspace, struct MMTk_ractor_cache *
 }
 
 VALUE
-rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size)
+rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size, size_t *actual_alloc_size)
 {
 #define MMTK_ALLOCATION_SEMANTICS_DEFAULT 0
     struct objspace *objspace = objspace_ptr;
@@ -916,6 +910,7 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags
             break;
         }
     }
+    *actual_alloc_size = alloc_size;
 
     if (objspace->gc_stress) {
         mmtk_handle_user_collection_request(ractor_cache, false, false);
@@ -962,10 +957,10 @@ rb_gc_impl_obj_slot_size(VALUE obj)
 }
 
 size_t
-rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size)
+rb_gc_impl_size_slot_size(void *objspace_ptr, size_t size)
 {
     for (int i = 0; i < MMTK_HEAP_COUNT; i++) {
-        if (size <= heap_sizes[i]) return i;
+        if (size <= heap_sizes[i]) return heap_sizes[i];
     }
 
     rb_bug("size too big");

gc/wbcheck/wbcheck.c

@@ -536,12 +536,6 @@ rb_gc_impl_init(void)
     // Stub implementation
 }
 
-size_t *
-rb_gc_impl_heap_sizes(void *objspace_ptr)
-{
-    return heap_sizes;
-}
-
 // Shutdown
 void
 rb_gc_impl_shutdown_free_objects(void *objspace_ptr)
@@ -1075,7 +1069,7 @@ lock_and_maybe_gc(void *objspace_ptr)
 }
 
 VALUE
-rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size)
+rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size, size_t *actual_alloc_size)
 {
     unsigned int lev = RB_GC_VM_LOCK();
     rb_gc_vm_barrier();
@@ -1084,12 +1078,14 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags
     maybe_gc(objspace_ptr);
 
     // Ensure minimum allocation size of BASE_SLOT_SIZE
-    alloc_size = heap_sizes[rb_gc_impl_heap_id_for_size(objspace_ptr, alloc_size)];
+    alloc_size = rb_gc_impl_size_slot_size(objspace_ptr, alloc_size);
 
     // Allocate memory for the object
     VALUE *mem = malloc(alloc_size);
     if (!mem) rb_bug("FIXME: malloc failed");
 
+    *actual_alloc_size = alloc_size;
+
     // Initialize the object
     VALUE obj = (VALUE)mem;
     RBASIC(obj)->flags = flags;
@@ -1119,10 +1115,10 @@ rb_gc_impl_obj_slot_size(VALUE obj)
 }
 
 size_t
-rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size)
+rb_gc_impl_size_slot_size(void *objspace_ptr, size_t size)
 {
     for (int i = 0; i < HEAP_COUNT; i++) {
-        if (size <= heap_sizes[i]) return i;
+        if (size <= heap_sizes[i]) return heap_sizes[i];
     }
     rb_bug("size too big");
 }
@@ -1952,4 +1948,3 @@ rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj)
     }
     rb_gc_impl_copy_finalizer(objspace_ptr, dest, obj);
 }
-

imemo.c

@@ -133,7 +133,7 @@ rb_imemo_cdhash_new(size_t size, const struct st_hash_type *type)
 VALUE
 rb_imemo_fields_new(VALUE owner, shape_id_t shape_id, bool shareable)
 {
-    size_t capa = RSHAPE_CAPACITY(shape_id);
+    size_t capa = RSHAPE_CAPACITY(shape_id & ~SHAPE_ID_CAPACITY_MASK);
     size_t embedded_size = offsetof(struct rb_fields, as.embed) + capa * sizeof(VALUE);
     RUBY_ASSERT(rb_gc_size_allocatable_p(embedded_size));
     VALUE fields = rb_imemo_new(imemo_fields, owner, embedded_size, shareable);

internal/gc.h

@@ -205,8 +205,7 @@ void *rb_gc_ractor_cache_alloc(rb_ractor_t *ractor);
 void rb_gc_ractor_cache_free(void *cache);
 
 bool rb_gc_size_allocatable_p(size_t size);
-size_t *rb_gc_heap_sizes(void);
-size_t rb_gc_heap_id_for_size(size_t size);
+size_t rb_gc_size_slot_size(size_t size);
 
 void rb_gc_mark_and_move(VALUE *ptr);