Merge branch 'master' into neon-simd-parser

Author: samyron

CHANGES.md

@@ -2,6 +2,9 @@
 
 ### Unreleased
 
+* Add bnew `allow_duplicate_key` parsing options. By default a warning is now emitted when a duplicated key is encountered.
+  In `json 3.0` an error will be raised.
+
 ### 2025-05-23 (2.12.2)
 
 * Fix compiler optimization level.

ext/json/ext/generator/generator.c

@@ -311,7 +311,7 @@ static inline unsigned char search_escape_basic_neon(search_state *search)
         if (search->matches_mask > 0) {
             return neon_next_match(search);
         } else {
-            // neon_next_match will only advance search->ptr up to the last matching character. 
+            // neon_next_match will only advance search->ptr up to the last matching character.
             // Skip over any characters in the last chunk that occur after the last match.
             search->has_matches = false;
             search->ptr = search->chunk_end;
@@ -320,40 +320,40 @@ static inline unsigned char search_escape_basic_neon(search_state *search)
 
     /*
     * The code below implements an SIMD-based algorithm to determine if N bytes at a time
-    * need to be escaped. 
-    * 
+    * need to be escaped.
+    *
     * Assume the ptr = "Te\sting!" (the double quotes are included in the string)
-    * 
+    *
     * The explanation will be limited to the first 8 bytes of the string for simplicity. However
     * the vector insructions may work on larger vectors.
-    * 
+    *
     * First, we load three constants 'lower_bound', 'backslash' and 'dblquote" in vector registers.
-    * 
-    * lower_bound: [20 20 20 20 20 20 20 20] 
-    * backslash:   [5C 5C 5C 5C 5C 5C 5C 5C] 
-    * dblquote:    [22 22 22 22 22 22 22 22] 
-    * 
-    * Next we load the first chunk of the ptr: 
+    *
+    * lower_bound: [20 20 20 20 20 20 20 20]
+    * backslash:   [5C 5C 5C 5C 5C 5C 5C 5C]
+    * dblquote:    [22 22 22 22 22 22 22 22]
+    *
+    * Next we load the first chunk of the ptr:
     * [22 54 65 5C 73 74 69 6E] ("  T  e  \  s  t  i  n)
-    * 
+    *
     * First we check if any byte in chunk is less than 32 (0x20). This returns the following vector
     * as no bytes are less than 32 (0x20):
     * [0 0 0 0 0 0 0 0]
-    * 
+    *
     * Next, we check if any byte in chunk is equal to a backslash:
     * [0 0 0 FF 0 0 0 0]
-    * 
+    *
     * Finally we check if any byte in chunk is equal to a double quote:
-    * [FF 0 0 0 0 0 0 0] 
-    * 
+    * [FF 0 0 0 0 0 0 0]
+    *
     * Now we have three vectors where each byte indicates if the corresponding byte in chunk
     * needs to be escaped. We combine these vectors with a series of logical OR instructions.
     * This is the needs_escape vector and it is equal to:
-    * [FF 0 0 FF 0 0 0 0] 
-    * 
+    * [FF 0 0 FF 0 0 0 0]
+    *
     * Next we compute the bitwise AND between each byte and 0x1 and compute the horizontal sum of
     * the values in the vector. This computes how many bytes need to be escaped within this chunk.
-    * 
+    *
     * Finally we compute a mask that indicates which bytes need to be escaped. If the mask is 0 then,
     * no bytes need to be escaped and we can continue to the next chunk. If the mask is not 0 then we
     * have at least one byte that needs to be escaped.
@@ -371,16 +371,16 @@ static inline unsigned char search_escape_basic_neon(search_state *search)
         search->matches_mask = mask;
         return neon_next_match(search);
     }
-    
-    // There are fewer than 16 bytes left. 
+
+    // There are fewer than 16 bytes left.
     unsigned long remaining = (search->end - search->ptr);
     if (remaining >= SIMD_MINIMUM_THRESHOLD) {
         char *s = copy_remaining_bytes(search, sizeof(uint8x16_t), remaining);
 
         uint64_t mask = compute_chunk_mask_neon(s);
 
         if (!mask) {
-            // Nothing to escape, ensure search_flush doesn't do anything by setting 
+            // Nothing to escape, ensure search_flush doesn't do anything by setting
             // search->cursor to search->ptr.
             fbuffer_consumed(search->buffer, remaining);
             search->ptr = search->end;
@@ -437,7 +437,7 @@ static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(se
         if (search->matches_mask > 0) {
             return sse2_next_match(search);
         } else {
-            // sse2_next_match will only advance search->ptr up to the last matching character. 
+            // sse2_next_match will only advance search->ptr up to the last matching character.
             // Skip over any characters in the last chunk that occur after the last match.
             search->has_matches = false;
             if (RB_UNLIKELY(search->chunk_base + sizeof(__m128i) >= search->end)) {
@@ -457,15 +457,15 @@ static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(se
         return sse2_next_match(search);
     }
 
-    // There are fewer than 16 bytes left. 
+    // There are fewer than 16 bytes left.
     unsigned long remaining = (search->end - search->ptr);
     if (remaining >= SIMD_MINIMUM_THRESHOLD) {
         char *s = copy_remaining_bytes(search, sizeof(__m128i), remaining);
 
         int needs_escape_mask = compute_chunk_mask_sse2(s);
 
         if (needs_escape_mask == 0) {
-            // Nothing to escape, ensure search_flush doesn't do anything by setting 
+            // Nothing to escape, ensure search_flush doesn't do anything by setting
             // search->cursor to search->ptr.
             fbuffer_consumed(search->buffer, remaining);
             search->ptr = search->end;

ext/json/ext/parser/parser.c

@@ -37,7 +37,7 @@ static ID i_chr, i_aset, i_aref,
           i_leftshift, i_new, i_try_convert, i_uminus, i_encode;
 
 static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_symbolize_names, sym_freeze,
-             sym_decimal_class, sym_on_load;
+             sym_decimal_class, sym_on_load, sym_allow_duplicate_key;
 
 static int binary_encindex;
 static int utf8_encindex;
@@ -365,10 +365,17 @@ static int convert_UTF32_to_UTF8(char *buf, uint32_t ch)
     return len;
 }
 
+enum duplicate_key_action {
+    JSON_DEPRECATED = 0,
+    JSON_IGNORE,
+    JSON_RAISE,
+};
+
 typedef struct JSON_ParserStruct {
     VALUE on_load_proc;
     VALUE decimal_class;
     ID decimal_method_id;
+    enum duplicate_key_action on_duplicate_key;
     int max_nesting;
     bool allow_nan;
     bool allow_trailing_comma;
@@ -388,15 +395,8 @@ typedef struct JSON_ParserStateStruct {
     int current_nesting;
 } JSON_ParserState;
 
-
-#define PARSE_ERROR_FRAGMENT_LEN 32
-#ifdef RBIMPL_ATTR_NORETURN
-RBIMPL_ATTR_NORETURN()
-#endif
-static void raise_parse_error(const char *format, JSON_ParserState *state)
+static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out)
 {
-    unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3];
-
     const char *cursor = state->cursor;
     long column = 0;
     long line = 1;
@@ -413,6 +413,27 @@ static void raise_parse_error(const char *format, JSON_ParserState *state)
             line++;
         }
     }
+    *line_out = line;
+    *column_out = column;
+}
+
+static void emit_parse_warning(const char *message, JSON_ParserState *state)
+{
+    long line, column;
+    cursor_position(state, &line, &column);
+
+    rb_warn("%s at line %ld column %ld", message, line, column);
+}
+
+#define PARSE_ERROR_FRAGMENT_LEN 32
+#ifdef RBIMPL_ATTR_NORETURN
+RBIMPL_ATTR_NORETURN()
+#endif
+static void raise_parse_error(const char *format, JSON_ParserState *state)
+{
+    unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3];
+    long line, column;
+    cursor_position(state, &line, &column);
 
     const char *ptr = "EOF";
     if (state->cursor && state->cursor < state->end) {
@@ -809,11 +830,25 @@ static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig
     return array;
 }
 
-static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, long count)
+static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, size_t count)
 {
-    VALUE object = rb_hash_new_capa(count);
+    size_t entries_count = count / 2;
+    VALUE object = rb_hash_new_capa(entries_count);
     rb_hash_bulk_insert(count, rvalue_stack_peek(state->stack, count), object);
 
+    if (RB_UNLIKELY(RHASH_SIZE(object) < entries_count)) {
+        switch (config->on_duplicate_key) {
+            case JSON_IGNORE:
+                break;
+            case JSON_DEPRECATED:
+                emit_parse_warning("detected duplicate keys in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`", state);
+                break;
+            case JSON_RAISE:
+                raise_parse_error("duplicate key", state);
+                break;
+        }
+    }
+
     rvalue_stack_pop(state->stack, count);
 
     if (config->freeze) {
@@ -1101,6 +1136,8 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
             break;
         }
         case '{': {
+            const char *object_start_cursor = state->cursor;
+
             state->cursor++;
             json_eat_whitespace(state);
             long stack_head = state->stack->head;
@@ -1135,8 +1172,15 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
                     if (*state->cursor == '}') {
                         state->cursor++;
                         state->current_nesting--;
-                        long count = state->stack->head - stack_head;
-                        return json_push_value(state, config, json_decode_object(state, config, count));
+                        size_t count = state->stack->head - stack_head;
+
+                        // Temporary rewind cursor in case an error is raised
+                        const char *final_cursor = state->cursor;
+                        state->cursor = object_start_cursor;
+                        VALUE object = json_decode_object(state, config, count);
+                        state->cursor = final_cursor;
+
+                        return json_push_value(state, config, object);
                     }
 
                     if (*state->cursor == ',') {
@@ -1225,6 +1269,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
     else if (key == sym_symbolize_names)      { config->symbolize_names = RTEST(val); }
     else if (key == sym_freeze)               { config->freeze = RTEST(val); }
     else if (key == sym_on_load)              { config->on_load_proc = RTEST(val) ? val : Qfalse; }
+    else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
     else if (key == sym_decimal_class)        {
         if (RTEST(val)) {
             if (rb_respond_to(val, i_try_convert)) {
@@ -1441,6 +1486,7 @@ void Init_parser(void)
     sym_freeze = ID2SYM(rb_intern("freeze"));
     sym_on_load = ID2SYM(rb_intern("on_load"));
     sym_decimal_class = ID2SYM(rb_intern("decimal_class"));
+    sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key"));
 
     i_chr = rb_intern("chr");
     i_aset = rb_intern("[]=");

java/src/json/ext/OptionsReader.java

@@ -53,6 +53,13 @@ IRubyObject get(String key) {
         return opts == null ? null : opts.fastARef(runtime.newSymbol(key));
     }
 
+    boolean hasKey(String key) {
+        if (opts == null) {
+            return false;
+        }
+        return opts.hasKey(runtime.newSymbol(key));
+    }
+
     boolean getBool(String key, boolean defaultValue) {
         IRubyObject value = get(key);
         return value == null ? defaultValue : value.isTrue();