add '--ratio' option to change threshold of collection ratio

In addition, change the default value to 0.5.

Signed-off-by: Daijiro Fukuda <fukuda@clear-code.com>

Author: daipom

README.md

@@ -161,6 +161,13 @@ If you use [follow_inodes](https://docs.fluentd.org/input/tail#follow_inodes), t
 $ tailcheck --follow_inodes /var/log/td-agent/pos/secure
 ```
 
+You can change the minimum ratio of collection of each target log file by specify `--ratio DECIMAL`.
+By default (`0.5`), the command detects log files that have not been collected up to 50% of the filesize.
+
+```
+$ tailcheck --ratio 0.7 /var/log/td-agent/pos/secure
+```
+
 ### Result example
 
 #### No anomalies found
@@ -211,9 +218,9 @@ Check /path/to/pos.
 Done duplication check for 2 PosEntries.
 Done collection ratio check for 2 files.
 Collection ratio of some files are too low. Collection of those files may not be keeping up. Or it may have stopped with some anomalies. This can be a known log missing issue of the follow_inodes feature that was fixed in Fluentd v1.16.2 (fluent-package v5.0.0, td-agent v4.5.1). If you are using any version older than these, updating Fluentd will resolve the issue.
-Filepaths with too low collection ratio (threshold: 0.8):
-  /test/bar.log (ratio: 0.7)
-  /test/foo.log (ratio: 0.7)
+Filepaths with too low collection ratio (threshold: 0.5):
+  /test/bar.log (ratio: 0.2)
+  /test/foo.log (ratio: 0.1)
 
 All check completed.
 Some anomalies are found. Please check whether there is any log loss.
@@ -222,9 +229,9 @@ Some anomalies are found. Please check whether there is any log loss.
 In this case, collection ratio of some target files are too low.
 Collection of those files may have stopped or may not be keeping up.
 
-> /test/bar.log (ratio: 0.7)
+> /test/bar.log (ratio: 0.2)
 
-This means that only 70% of the data of the file is collected for the filesize.
+This means that only 20% of the data of the file is collected for the filesize.
 If it is not keeping up temporarily, then it is no problem.
 If this is always the case, or if collection has stopped completely, then log missing may occur.
 

lib/fluent/tail_checker/collection_ratio_checker.rb

@@ -31,10 +31,10 @@
 module Fluent
   module TailChecker
     class CollectionRatioChecker
-      def initialize(posfile, follow_inodes)
+      def initialize(posfile, follow_inodes, collection_ratio_threshold)
         @posfile = posfile
         @follow_inodes = follow_inodes
-        @collection_ratio_threshold = 0.8
+        @collection_ratio_threshold = collection_ratio_threshold
       end
 
       def check

lib/fluent/tail_checker/tail_check.rb

@@ -26,6 +26,7 @@ class TailCheck
       def initialize
         @pos_filepaths = []
         @follow_inodes = false
+        @collection_ratio_threshold = 0.5
       end
 
       def run(argv=ARGV)
@@ -51,8 +52,20 @@ def parse_command_line(argv)
         parser.on("--follow_inodes", "Check the specified pos files with the condition that the follow_inodes feature is enabled.", "Default: Disabled") do
           @follow_inodes = true
         end
+        parser.on("--ratio NUM", Float, "Minimum ratio of collection of each target log file to accept.", "Default: #{@collection_ratio_threshold}") do |v|
+          @collection_ratio_threshold = v
+        end
+
+        begin
+          @pos_filepaths = parser.parse(argv)
 
-        @pos_filepaths = parser.parse(argv)
+          if @collection_ratio_threshold < 0 or @collection_ratio_threshold > 1
+            raise OptionParser::InvalidArgument, "--ratio #{@collection_ratio_threshold} must be an decimal from 0 to 1."
+          end
+        rescue OptionParser::ParseError => e
+          $stderr.puts e, "", parser.help, ""
+          raise
+        end
       end
 
       def validate_paths(paths)
@@ -82,7 +95,7 @@ def check
           next if pos_file.nil?
 
           succeeded = DuplicatedPosChecker.new(pos_file, @follow_inodes).check && succeeded
-          succeeded = CollectionRatioChecker.new(pos_file, @follow_inodes).check && succeeded
+          succeeded = CollectionRatioChecker.new(pos_file, @follow_inodes, @collection_ratio_threshold).check && succeeded
         end
 
         puts "\nAll check completed."

test/fluent/tail_checker/collection_ratio_checker_test.rb

@@ -4,16 +4,16 @@
 
 class Fluent::TailChecker::CollectionRatioCheckerTest < Test::Unit::TestCase
   data("Normal ratio", [["test/data/log/foo.log", "test/data/log/bar.log"], 0.9, false, true])
-  data("Too low ratio", [["test/data/log/foo.log", "test/data/log/bar.log"], 0.7, false, false])
+  data("Too low ratio", [["test/data/log/foo.log", "test/data/log/bar.log"], 0.3, false, false])
   data("Normal ratio with follow_inodes", [["test/data/log/foo.log", "test/data/log/bar.log"], 0.9, true, true])
-  data("Too low ratio with follow_inodes", [["test/data/log/foo.log", "test/data/log/foo.log.1"], 0.7, true, false])
+  data("Too low ratio with follow_inodes", [["test/data/log/foo.log", "test/data/log/foo.log.1"], 0.3, true, false])
   test "Check should return false when too low collection ratio is detected" do |(paths, ratio, follow_inodes, expected)|
     pos_entries = paths.map do |path|
       stat = Fluent::FileWrapper.stat(path)
       Fluent::TailChecker::PosEntry.new(path, stat.size * ratio, stat.ino)
     end
     pos_file = Fluent::TailChecker::PosFile.new(pos_entries)
-    checker = Fluent::TailChecker::CollectionRatioChecker.new(pos_file, follow_inodes)
+    checker = Fluent::TailChecker::CollectionRatioChecker.new(pos_file, follow_inodes, 0.5)
 
     result = checker.check
 

test/fluent/tail_checker_test.rb

@@ -14,14 +14,14 @@ class Fluent::TailCheckerTest < Test::Unit::TestCase
       "Minimum",
       [
         ["/path/to/pos"],
-        { pos_filepaths: ["/path/to/pos"], follow_inodes: false },
+        { pos_filepaths: ["/path/to/pos"], follow_inodes: false, collection_ratio_threshold: 0.5 },
       ]
     )
     data(
       "Full",
       [
-        ["--follow_inodes", "/path/to/pos", "/path/to/pos2"],
-        { pos_filepaths: ["/path/to/pos", "/path/to/pos2"], follow_inodes: true },
+        ["--follow_inodes", "--ratio", "0.7", "/path/to/pos", "/path/to/pos2"],
+        { pos_filepaths: ["/path/to/pos", "/path/to/pos2"], follow_inodes: true, collection_ratio_threshold: 0.7 },
       ]
     )
     test "Correct args" do |(args, expected)|
@@ -32,15 +32,17 @@ class Fluent::TailCheckerTest < Test::Unit::TestCase
       result = {
         pos_filepaths: tail_check.instance_variable_get(:@pos_filepaths),
         follow_inodes: tail_check.instance_variable_get(:@follow_inodes),
+        collection_ratio_threshold: tail_check.instance_variable_get(:@collection_ratio_threshold),
       }
       assert_equal(expected, result)
     end
 
     data("Invalid options", ["--foo", "/path/to/pos"])
+    data("--ratio: invalid value: ", ["--ratio", "70", "/path/to/pos"])
     test "Raise error for invalid options" do |args|
       tail_check = Fluent::TailChecker::TailCheck.new
 
-      assert_raise(OptionParser::InvalidOption) do
+      assert_raise do
         tail_check.parse_command_line(args)
       end
     end
@@ -84,7 +86,7 @@ class Fluent::TailCheckerTest < Test::Unit::TestCase
     end
 
     data("Acceptable ratio", [0.9, true])
-    data("Too low ratio", [0.7, false])
+    data("Too low ratio", [0.3, false])
     test "Return false when too low collection ratio is detected" do |(stub_ratio, expected)|
       any_instance_of(Fluent::TailChecker::CollectionRatioChecker) do |checker|
         mock(checker).get_file_size_from_path(anything).at_least(1) do |pos_entry|
@@ -104,7 +106,7 @@ class Fluent::TailCheckerTest < Test::Unit::TestCase
     end
 
     data("Acceptable ratio", [0.9, true])
-    data("Too low ratio", [0.7, false])
+    data("Too low ratio", [0.3, false])
     test "Return false when too low collection ratio is detected (follow_inodes)" do |(stub_ratio, expected)|
       any_instance_of(Fluent::TailChecker::CollectionRatioChecker) do |checker|
         mock(checker).get_file_size_from_path.never