Set allow_duplicate_key: true for all JSON.parse call (#5410)

Watson1978 · web-flow · commit 99c7074fcdcc · 2026-07-02T18:14:48.000+09:00
**Which issue(s) this PR fixes**: Fixes # **What this PR does / why we need it**: The json gem emits a deprecation warning for duplicate keys in JSON objects and will raise `JSON::ParserError` in json 3.0 unless `allow_duplicate_key: true` is specified. Fluentd has historically accepted duplicate keys silently (last value wins) on every path: Yajl-based stream parsers, plain `JSON.parse` before json 2.10, and Oj. To preserve this behavior uniformly regardless of the json gem version, introduce `Fluent::DEFAULT_JSON_PARSE_OPTIONS` and pass it to all `JSON.parse`. Ref. https://github.com/ruby/json/blob/master/CHANGES.md#2025-09-18-2140 **Docs Changes**: N/A **Release Note**: * Set `allow_duplicate_key: true` for all `JSON.parse` call Signed-off-by: Shizuo Fujita <fujita@clear-code.com>
diff --git a/lib/fluent/command/cat.rb b/lib/fluent/command/cat.rb
@@ -326,7 +326,7 @@ def abort_message(time, record)
 when 'json'
   begin
     while line = $stdin.gets
-      record = JSON.parse(line)
+      record = JSON.parse(line, Fluent::DEFAULT_JSON_PARSE_OPTIONS)
       w.write(record)
     end
   rescue
diff --git a/lib/fluent/config/literal_parser.rb b/lib/fluent/config/literal_parser.rb
@@ -21,6 +21,7 @@
 require 'socket'
 require 'ripper'
 
+require 'fluent/env'
 require 'fluent/config/basic_parser'
 
 module Fluent
@@ -241,7 +242,7 @@ def scan_json(is_array)
             # '{"foo":"bar", #' -> '{"foo":"bar"}' (to check)
             parsed = nil
             begin
-              parsed = JSON.parse(buffer + line_buffer.rstrip.sub(/,$/, '') + (is_array ? "]" : "}"))
+              parsed = JSON.parse(buffer + line_buffer.rstrip.sub(/,$/, '') + (is_array ? "]" : "}"), Fluent::DEFAULT_JSON_PARSE_OPTIONS)
             rescue JSON::ParserError
               # This '#' is in json string literals
             end
@@ -276,7 +277,7 @@ def scan_json(is_array)
 
           line_buffer << char
           begin
-            result = JSON.parse(buffer + line_buffer)
+            result = JSON.parse(buffer + line_buffer, Fluent::DEFAULT_JSON_PARSE_OPTIONS)
           rescue JSON::ParserError
             # Incomplete json string yet
           end
diff --git a/lib/fluent/config/types.rb b/lib/fluent/config/types.rb
@@ -201,7 +201,7 @@ def self.hash_value(val, opts = {}, name = nil)
       return nil if val.nil?
 
       param = if val.is_a?(String)
-                val.start_with?('{') ? JSON.parse(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
+                val.start_with?('{') ? JSON.parse(val, Fluent::DEFAULT_JSON_PARSE_OPTIONS) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
               else
                 val
               end
@@ -228,7 +228,7 @@ def self.array_value(val, opts = {}, name = nil)
       return nil if val.nil?
 
       param = if val.is_a?(String)
-                val.start_with?('[') ? JSON.parse(val) : val.strip.split(/\s*,\s*/)
+                val.start_with?('[') ? JSON.parse(val, Fluent::DEFAULT_JSON_PARSE_OPTIONS) : val.strip.split(/\s*,\s*/)
               else
                 val
               end
diff --git a/lib/fluent/configurable.rb b/lib/fluent/configurable.rb
@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/env'
 require 'fluent/config/configure_proxy'
 require 'fluent/config/section'
 require 'fluent/config/error'
diff --git a/lib/fluent/daemon.rb b/lib/fluent/daemon.rb
@@ -5,9 +5,10 @@
 $LOAD_PATH << File.expand_path(File.join(here, '..'))
 
 require 'serverengine'
+require 'fluent/env'
 require 'fluent/supervisor'
 
 server_module = Fluent.const_get(ARGV[0])
 worker_module = Fluent.const_get(ARGV[1])
-params = JSON.parse(ARGV[2])
+params = JSON.parse(ARGV[2], Fluent::DEFAULT_JSON_PARSE_OPTIONS)
 ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.serverengine_config(params) }
diff --git a/lib/fluent/env.rb b/lib/fluent/env.rb
@@ -26,6 +26,7 @@ module Fluent
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
   DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env
+  DEFAULT_JSON_PARSE_OPTIONS = { allow_duplicate_key: true }.freeze
   DEFAULT_DIR_PERMISSION = 0755
   DEFAULT_FILE_PERMISSION = 0644
   INSTANCE_ID = ENV['FLUENT_INSTANCE_ID'] || SecureRandom.uuid
diff --git a/lib/fluent/plugin/filter_record_transformer.rb b/lib/fluent/plugin/filter_record_transformer.rb
@@ -18,6 +18,7 @@
 require 'json'
 require 'ostruct'
 
+require 'fluent/env'
 require 'fluent/plugin/filter'
 require 'fluent/config/error'
 require 'fluent/event'
@@ -116,7 +117,7 @@ def filter_stream(tag, es)
 
     def parse_value(value_str)
       if value_str.start_with?('{', '[')
-        JSON.parse(value_str)
+        JSON.parse(value_str, Fluent::DEFAULT_JSON_PARSE_OPTIONS)
       else
         value_str
       end
diff --git a/lib/fluent/plugin/in_monitor_agent.rb b/lib/fluent/plugin/in_monitor_agent.rb
@@ -16,6 +16,7 @@
 
 require 'json'
 
+require 'fluent/env'
 require 'fluent/config/types'
 require 'fluent/plugin/input'
 require 'fluent/plugin/output'
@@ -101,7 +102,7 @@ def render_json(obj, code: 200, pretty_json: nil)
       end
 
       def render_ltsv(obj, code: 200)
-        normalized = JSON.parse(obj.to_json)
+        normalized = JSON.parse(obj.to_json, Fluent::DEFAULT_JSON_PARSE_OPTIONS)
         text = ''
         normalized.each do |hash|
           row = []
diff --git a/lib/fluent/plugin/in_sample.rb b/lib/fluent/plugin/in_sample.rb
@@ -16,6 +16,7 @@
 
 require 'json'
 
+require 'fluent/env'
 require 'fluent/plugin/input'
 require 'fluent/config/error'
 
@@ -44,7 +45,7 @@ class SampleInput < Input
     desc "The sample data to be generated. An array of JSON hashes or a single JSON hash."
     config_param :sample, alias: :dummy, default: [{"message" => "sample"}] do |val|
       begin
-        parsed = JSON.parse(val)
+        parsed = JSON.parse(val, Fluent::DEFAULT_JSON_PARSE_OPTIONS)
       rescue JSON::ParserError => ex
         # Fluent::ConfigParseError, "got incomplete JSON" will be raised
         # at literal_parser.rb with --use-v1-config, but I had to
diff --git a/lib/fluent/plugin/parser_json.rb b/lib/fluent/plugin/parser_json.rb
@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/env'
 require 'fluent/plugin/parser'
 require 'fluent/time'
 require 'fluent/oj_options'
@@ -38,6 +39,10 @@ class JSONParser < Parser
 
       config_set_default :time_type, :float
 
+      # Use a shared proc rather than a per-call lambda so that
+      # configure_json_parser returns the same object every time.
+      JSON_PARSE_PROC = ->(text) { JSON.parse(text, Fluent::DEFAULT_JSON_PARSE_OPTIONS) }
+
       def configure(conf)
         if conf.has_key?('time_format')
           conf['time_type'] ||= 'string'
@@ -54,7 +59,7 @@ def configure_json_parser(name)
 
           log&.info "Oj is not installed, and failing back to JSON for json parser"
           configure_json_parser(:json)
-        when :json then [JSON.method(:parse), JSON::ParserError]
+        when :json then [JSON_PARSE_PROC, JSON::ParserError]
         when :yajl then [Yajl.method(:load), Yajl::ParseError]
         else
           raise "BUG: unknown json parser specified: #{name}"
diff --git a/lib/fluent/plugin/sd_file.rb b/lib/fluent/plugin/sd_file.rb
@@ -16,6 +16,7 @@
 
 require 'cool.io'
 
+require 'fluent/env'
 require 'fluent/plugin_helper'
 require 'fluent/plugin/service_discovery'
 
@@ -75,7 +76,7 @@ def parser
             -> (v) { YAML.safe_load(v).map }
           when :json
             require 'json'
-            -> (v) { JSON.parse(v) }
+            -> (v) { JSON.parse(v, Fluent::DEFAULT_JSON_PARSE_OPTIONS) }
           end
       end
 
diff --git a/lib/fluent/plugin/storage_local.rb b/lib/fluent/plugin/storage_local.rb
@@ -90,7 +90,7 @@ def configure(conf)
                 log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
                 return
               end
-              data = JSON.parse(data)
+              data = JSON.parse(data, Fluent::DEFAULT_JSON_PARSE_OPTIONS)
               raise Fluent::ConfigError, "Invalid contents (not object) in plugin storage file: '#{@path}'" unless data.is_a?(Hash)
             rescue => e
               log.error "failed to read data from plugin storage file", path: @path, error: e
@@ -114,7 +114,7 @@ def load
         return unless File.exist?(@path)
         begin
           json_string = File.open(@path, 'r:utf-8:utf-8'){ |io| io.read }
-          json = JSON.parse(json_string)
+          json = JSON.parse(json_string, Fluent::DEFAULT_JSON_PARSE_OPTIONS)
           unless json.is_a?(Hash)
             log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
             log.debug "broken content", content: json_string
diff --git a/test/plugin/test_parser_json.rb b/test/plugin/test_parser_json.rb
@@ -10,7 +10,7 @@ def setup
 
   sub_test_case "configure_json_parser" do
     data("oj", [:oj, [Oj.method(:load), Oj::ParseError]])
-    data("json", [:json, [JSON.method(:parse), JSON::ParserError]])
+    data("json", [:json, [Fluent::Plugin::JSONParser::JSON_PARSE_PROC, JSON::ParserError]])
     data("yajl", [:yajl, [Yajl.method(:load), Yajl::ParseError]])
     def test_return_each_loader((input, expected_return))
       result = @parser.instance.configure_json_parser(input)
@@ -28,7 +28,7 @@ def test_fall_back_oj_to_json_if_oj_not_available
 
       result = @parser.instance.configure_json_parser(:oj)
 
-      assert_equal [JSON.method(:parse), JSON::ParserError], result
+      assert_equal [Fluent::Plugin::JSONParser::JSON_PARSE_PROC, JSON::ParserError], result
       logs = @parser.logs.collect do |log|
         log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
       end
@@ -52,6 +52,16 @@ def test_parse(data)
     }
   end
 
+  data('oj' => 'oj', 'json' => 'json', 'yajl' => 'yajl')
+  def test_parse_with_duplicated_key(data)
+    @parser.configure('json_parser' => data)
+    assert_nothing_raised do
+      @parser.instance.parse('{"k":"a","k":"b"}') { |time, record|
+        assert_equal('b', record['k'])
+      }
+    end
+  end
+
   data('oj' => 'oj', 'yajl' => 'yajl')
   def test_parse_with_large_float(data)
     @parser.configure('json_parser' => data)

Original file line number	Diff line number	Diff line change
`@@ -14,6 +14,7 @@`
`14`	`14`	`# limitations under the License.`
`15`	`15`	`#`
`16`	`16`
	`17`	`+require 'fluent/env'`
`17`	`18`	`require 'fluent/config/configure_proxy'`
`18`	`19`	`require 'fluent/config/section'`
`19`	`20`	`require 'fluent/config/error'`