dont use mrb_alloca

Author: Asmod4n

build_config.rb

@@ -4,7 +4,7 @@ def for_windows?
         ('A'..'Z').to_a.any? { |vol| Dir.exist?("#{vol}:") }
     end
     unless for_windows?
-        #conf.enable_sanitizer "address,undefined"
+        conf.enable_sanitizer "address,undefined"
     end
     conf.cxx.flags << '-fno-omit-frame-pointer' << '-g3' << '-ggdb3' << '-Og'
     conf.cc.flags << '-fno-omit-frame-pointer' << '-g3' << '-ggdb3' << '-Og'

fuzz/bintest/mruby-cbor-fuzzer.rb

@@ -12,11 +12,12 @@
   cmd = cmd_list('mruby-cbor-fuzzer') + [
     CORPUS_DIR,
     "-dict=#{DICT_FILE}",
-    "-jobs=7",
+    "-jobs=3",
     "-artifact_prefix=#{findings_dir}/",
     "-max_len=65536",
-     "-ignore_ooms=0",
-     "-use_value_profile=1"
+     "-ignore_ooms=1",
+     "-use_value_profile=1",
+     "-rss_limit_mb=3072"
   ]
 
   Dir.chdir(findings_dir) do

fuzz/corpus/659e18039fb7c9fa02fa1ee081216907b8349e30

@@ -26,7 +26,7 @@ MRB_END_DECL
 /* Configurable CBOR recursion depth limits */
 #ifndef CBOR_MAX_DEPTH
   #if defined(MRB_PROFILE_MAIN) || defined(MRB_PROFILE_HIGH)
-    #define CBOR_MAX_DEPTH 512
+    #define CBOR_MAX_DEPTH 128
   #elif defined(MRB_PROFILE_BASELINE)
     #define CBOR_MAX_DEPTH 64
   #else
@@ -964,16 +964,26 @@ encode_bignum(CborWriter *w, mrb_value obj)
   }
 
   mrb_value mag = mrb_bint_abs(mrb, obj);
+  mrb_gc_protect(mrb, mag);
   if (sign < 0) {
     mrb_value one = mrb_fixnum_value(1);
     mag = mrb_bint_sub(mrb, mag, one);
   }
+  mrb_gc_protect(mrb, mag);
 
   mrb_value hex = mrb_bint_to_s(mrb, mag, 16);
-  mrb_gc_register(mrb, hex);
-  char *p = RSTRING_PTR(hex);
+  mrb_gc_protect(mrb, hex);
+
   mrb_int len = RSTRING_LEN(hex);
+  // Copy into a C buffer now, before any further mruby allocations
+  char *hbuf = (char*)mrb_malloc(mrb, len + 2); // +2 for odd-pad + NUL
+  memcpy(hbuf, RSTRING_PTR(hex), len);
+  hbuf[len] = '\0';
+
+  // hex no longer needed as a live mruby object
+  mrb_gc_arena_restore(mrb, idx);
 
+  char *p = hbuf;
   while (len > 0 && *p == '0') { p++; len--; }
 
   if (len == 0) {
@@ -982,8 +992,8 @@ encode_bignum(CborWriter *w, mrb_value obj)
     encode_len(w, 2, 1);
     uint8_t zero = 0;
     cbor_writer_write(w, &zero, 1);
-    mrb_gc_unregister(mrb, hex);
     mrb_gc_arena_restore(mrb, idx);
+    mrb_free(mrb, hbuf);
     return;
   }
 
@@ -996,10 +1006,11 @@ encode_bignum(CborWriter *w, mrb_value obj)
 
   if (odd) { memmove(p + 1, p, len); p[0] = '0'; }
 
-  uint8_t *out = (uint8_t*)mrb_alloca(mrb, byte_len);
+  uint8_t *out = (uint8_t*)mrb_malloc(mrb, byte_len);
   hex_decode_scalar(out, p, byte_len);
   cbor_writer_write(w, out, (size_t)byte_len);
-  mrb_gc_unregister(mrb, hex);
+  mrb_free(mrb, hbuf);
+  mrb_free(mrb, out);
   mrb_gc_arena_restore(mrb, idx);
 }
 #endif