Type stats follow renaming

Author: marcotc

.github/scripts/typing_stats_compare.rb

@@ -7,6 +7,40 @@
 head_stats = JSON.parse(File.read(ENV["CURRENT_STATS_PATH"]), symbolize_names: true)
 base_stats = JSON.parse(File.read(ENV["BASE_STATS_PATH"]), symbolize_names: true)
 
+# Parse renames from Git.
+RENAMED_PATHS = File.readlines(ENV["RENAMED_PATHS_PATH"], chomp: true).each_with_object({}) do |line, result|
+  _, base_path, head_path = line.split("\t", 3)
+  result[head_path] = base_path
+end
+
+# Normalizes comparison keys so findings follow renames.
+# Example: on rename from base "base.rbs" to head "head.rbs", both paths compare as "base.rbs".
+def comparison_key(item)
+  key = item[:comparison_key].dup
+  # Normalize key as base path.
+  key[:path] = RENAMED_PATHS.fetch(key[:path], key[:path])
+  key
+end
+
+# Compares keyed findings while preserving duplicate counts.
+def multiset_comparison(head_items, base_items)
+  unmatched_base = base_items.group_by { |item| comparison_key(item) }
+  added = []
+
+  head_items.each do |item|
+    base_matches = unmatched_base[comparison_key(item)]
+    if base_matches&.any?
+      # Consume one matching base finding so duplicate findings are counted independently.
+      base_matches.shift
+    else
+      added << item
+    end
+  end
+
+  removed = unmatched_base.values.flatten
+  [added, removed]
+end
+
 def format_for_code_block(data)
   data.map do |item|
     formatted_string = +"#{item[:path]}:#{item[:line]}"
@@ -146,8 +180,10 @@ def ignored_files_summary(head_stats, base_stats)
 end
 
 def steep_ignore_summary(head_stats, base_stats)
-  steep_ignore_added = head_stats[:steep_ignore_comments] - base_stats[:steep_ignore_comments]
-  steep_ignore_removed = base_stats[:steep_ignore_comments] - head_stats[:steep_ignore_comments]
+  steep_ignore_added, steep_ignore_removed = multiset_comparison(
+    head_stats[:steep_ignore_comments],
+    base_stats[:steep_ignore_comments]
+  )
 
   create_summary(
     added: steep_ignore_added,
@@ -157,10 +193,14 @@ def steep_ignore_summary(head_stats, base_stats)
 end
 
 def untyped_methods_summary(head_stats, base_stats)
-  untyped_methods_added = head_stats[:untyped_methods] - base_stats[:untyped_methods]
-  untyped_methods_removed = base_stats[:untyped_methods] - head_stats[:untyped_methods]
-  partially_typed_methods_added = head_stats[:partially_typed_methods] - base_stats[:partially_typed_methods]
-  partially_typed_methods_removed = base_stats[:partially_typed_methods] - head_stats[:partially_typed_methods]
+  untyped_methods_added, untyped_methods_removed = multiset_comparison(
+    head_stats[:untyped_methods],
+    base_stats[:untyped_methods]
+  )
+  partially_typed_methods_added, partially_typed_methods_removed = multiset_comparison(
+    head_stats[:partially_typed_methods],
+    base_stats[:partially_typed_methods]
+  )
   total_methods_base = base_stats[:typed_methods_size] + base_stats[:untyped_methods].size + base_stats[:partially_typed_methods].size
   total_methods_head = head_stats[:typed_methods_size] + head_stats[:untyped_methods].size + head_stats[:partially_typed_methods].size
   typed_methods_percentage_base = (base_stats[:typed_methods_size] / total_methods_base.to_f * 100).round(2)
@@ -180,10 +220,14 @@ def untyped_methods_summary(head_stats, base_stats)
 end
 
 def untyped_others_summary(head_stats, base_stats)
-  untyped_others_added = head_stats[:untyped_others] - base_stats[:untyped_others]
-  untyped_others_removed = base_stats[:untyped_others] - head_stats[:untyped_others]
-  partially_typed_others_added = head_stats[:partially_typed_others] - base_stats[:partially_typed_others]
-  partially_typed_others_removed = base_stats[:partially_typed_others] - head_stats[:partially_typed_others]
+  untyped_others_added, untyped_others_removed = multiset_comparison(
+    head_stats[:untyped_others],
+    base_stats[:untyped_others]
+  )
+  partially_typed_others_added, partially_typed_others_removed = multiset_comparison(
+    head_stats[:partially_typed_others],
+    base_stats[:partially_typed_others]
+  )
   total_others_base = base_stats[:typed_others_size] + base_stats[:untyped_others].size + base_stats[:partially_typed_others].size
   total_others_head = head_stats[:typed_others_size] + head_stats[:untyped_others].size + head_stats[:partially_typed_others].size
   typed_others_percentage_base = (base_stats[:typed_others_size] / total_others_base.to_f * 100).round(2)

.github/scripts/typing_stats_compute.rb

@@ -39,40 +39,63 @@
 
 total_files_size = Dir.glob("#{project.base_dir}/lib/**/*.rb").size
 
+def stats_item(path, line, line_content, comparison_key)
+  {
+    path: path.to_s,
+    line: line,
+    line_content: line_content,
+    comparison_key: comparison_key
+  }.compact
+end
+
+# Include the ignore comment and ignored source so line-only moves stay matched.
+def ignore_item(path, line, comment_source, ignored_source)
+  stats_item(path, line, nil, {type: "steep_ignore", path: path.to_s, source: comment_source, ignored_source: ignored_source})
+end
+
+# Include the enclosing namespace so equivalent RBS declarations in different owners stay distinct.
+def rbs_item(path, line, source, context)
+  stats_item(path, line, source, {type: "rbs_declaration", path: path.to_s, context: context, source: source})
+end
+
 # steep:ignore comments stats
 ignore_comments = loader.each_path_in_patterns(datadog_target.source_pattern).each_with_object([]) do |path, result|
-  buffer = ::Parser::Source::Buffer.new(path.to_s, 1, source: path.read)
+  source = path.read
+  source_lines = source.lines
+  buffer = ::Parser::Source::Buffer.new(path.to_s, 1, source: source)
   _, comments = ::Parser::Ruby25.new.parse_with_comments(buffer)
-  rbs_buffer = ::RBS::Buffer.new(name: path, content: path.read)
+  rbs_buffer = ::RBS::Buffer.new(name: path, content: source)
   comments.each do |comment|
     ignore = ::Steep::AST::Ignore.parse(comment, rbs_buffer)
     next if ignore.nil? || ignore.is_a?(::Steep::AST::Ignore::IgnoreEnd)
 
-    result << {
-      path: path.to_s,
-      line: ignore.line
-    }
+    ignored_source = source_lines[ignore.line - 1]&.strip
+    result << ignore_item(path, ignore.line, comment.loc.expression.source, ignored_source)
   end
 end
 
-def ast_traversal(declarations, result = {})
+# Collects declarations with their enclosing RBS namespace for stable comparison keys.
+# @param declarations [Array<RBS::AST::Declarations::Base, RBS::AST::Members::Base>]
+# @param result [Hash] accumulated methods and other declarations
+# @param context [Array<String>] enclosing Ruby namespace, e.g. ["Datadog", "Core"]
+def ast_traversal(declarations, result = {}, context = [])
   result[:methods] ||= []
   result[:others] ||= []
   declarations.each do |declaration|
     case declaration
     when ::RBS::AST::Declarations::Module,
          ::RBS::AST::Declarations::Class,
          ::RBS::AST::Declarations::Interface
-      ast_traversal(declaration.members, result)
+      ast_traversal(declaration.members, result, context + [declaration.name.to_s])
     when ::RBS::AST::Declarations::TypeAlias,
       ::RBS::AST::Declarations::Constant,
       ::RBS::AST::Declarations::Global,
       ::RBS::AST::Members::Var,
       ::RBS::AST::Members::Attribute
-      result[:others] << declaration
+      result[:others] << {declaration: declaration, context: context}
     # Only this one does not have a type field
     when ::RBS::AST::Members::MethodDefinition
-      result[:methods] << declaration
+      result[:methods] << {declaration: declaration, context: context}
     end
   end
   result
@@ -177,7 +200,9 @@ def is_typed?(type, initialize: false)
   _, _directives, declarations = ::RBS::Parser.parse_signature(buffer)
   filtered_declarations = ast_traversal(declarations)
 
-  filtered_declarations[:methods].each do |method|
+  filtered_declarations[:methods].each do |entry|
+    method = entry[:declaration]
+
     # Skip definitions with last comment line being `untyped:accept`
     if method.comment&.string&.end_with?("untyped:accept\n")
       typed_methods_size += 1
@@ -192,13 +217,15 @@ def is_typed?(type, initialize: false)
     when :typed
       typed_methods_size += 1
     when :untyped
-      untyped_methods << {path: sig_path.to_s, line: method.location.start_line, line_content: method.location.source}
+      untyped_methods << rbs_item(sig_path, method.location.start_line, method.location.source, entry[:context])
     when :partial
-      partially_typed_methods << {path: sig_path.to_s, line: method.location.start_line, line_content: method.location.source}
+      partially_typed_methods << rbs_item(sig_path, method.location.start_line, method.location.source, entry[:context])
     end
   end
 
-  filtered_declarations[:others].each do |declaration|
+  filtered_declarations[:others].each do |entry|
+    declaration = entry[:declaration]
+
     # Skip definitions with last comment line being `untyped:accept`
     if declaration.comment&.string&.end_with?("untyped:accept\n")
       typed_others_size += 1
@@ -209,9 +236,9 @@ def is_typed?(type, initialize: false)
     when :typed, nil
       typed_others_size += 1
     when :untyped
-      untyped_others << {path: sig_path.to_s, line: declaration.location.start_line, line_content: declaration.location.source}
+      untyped_others << rbs_item(sig_path, declaration.location.start_line, declaration.location.source, entry[:context])
     when :partial
-      partially_typed_others << {path: sig_path.to_s, line: declaration.location.start_line, line_content: declaration.location.source}
+      partially_typed_others << rbs_item(sig_path, declaration.location.start_line, declaration.location.source, entry[:context])
     end
   end
 end

.github/workflows/typing-stats.yml

@@ -44,6 +44,7 @@ jobs:
       - name: Checkout current branch
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
+          fetch-depth: 0
           persist-credentials: false
 
       - name: Set up Ruby
@@ -67,6 +68,10 @@ jobs:
           mkdir -p "${{ github.workspace }}/tmp"
           ruby ${{ runner.temp }}/typing_stats_compute.rb >> "${{ runner.temp }}/typing-stats-current.json"
 
+      - name: Collect renamed paths
+        # Lets the compare script match findings across file renames.
+        run: git diff --find-renames --diff-filter=R --name-status ${{ github.event.pull_request.base.sha }} HEAD >> "${{ runner.temp }}/typing-stats-renamed-paths.txt"
+
       - name: Checkout base branch
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
@@ -84,6 +89,7 @@ jobs:
         env:
           CURRENT_STATS_PATH: ${{ runner.temp }}/typing-stats-current.json
           BASE_STATS_PATH: ${{ runner.temp }}/typing-stats-base.json
+          RENAMED_PATHS_PATH: ${{ runner.temp }}/typing-stats-renamed-paths.txt
         run: ruby ${{ runner.temp }}/typing_stats_compare.rb >> "${{ runner.temp }}/typing-stats-compare.md"
 
       - name: Write comment

Matrixfile

@@ -8,7 +8,7 @@
     'core-old' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ 4.0'
   },
   # Rubocop not included in Ruby < 2.7 gemfiles
-  'custom_cop' => {
+  'tooling' => {
     '' => '❌ 2.5 / ❌ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ 4.0'
   },
   'core_with_libdatadog_api' => {

Rakefile

@@ -90,7 +90,7 @@ end
 desc 'Run RSpec'
 namespace :spec do
   # REMINDER: If adding a new task here, make sure also add it to the `Matrixfile`
-  task all: [:main, :benchmark, :custom_cop,
+  task all: [:main, :benchmark, :tooling,
     :graphql, :graphql_unified_trace_patcher, :graphql_trace_patcher, :graphql_tracing_patcher,
     :rails, :railsredis, :railsredis_activesupport, :railsactivejob,
     :elasticsearch, :http, :redis, :sidekiq, :sinatra, :hanami, :hanami_autoinstrument,
@@ -99,7 +99,10 @@ namespace :spec do
   desc '' # "Explicitly hiding from `rake -T`"
   RSpec::Core::RakeTask.new(:main) do |t, args|
     t.pattern = 'spec/**/*_spec.rb'
+
+    # Add new entries as new lines, for readable diffs.
     t.exclude_pattern = 'spec/**/{appsec/integration,contrib,benchmark,redis,auto_instrument,opentelemetry,open_feature,profiling,error_tracking,rubocop,ai_guard}/**/*_spec.rb,' \
+                        ' spec/tooling/**/*_spec.rb,' \
                         ' spec/**/{auto_instrument,opentelemetry,process,ai_guard}_spec.rb,' \
                         ' spec/**/*_rails_spec.rb,' \
                         ' spec/datadog/core/environment/execution_spec.rb,' \
@@ -115,8 +118,12 @@ namespace :spec do
     t.rspec_opts = args.to_a.join(' ')
   end
 
-  RSpec::Core::RakeTask.new(:custom_cop) do |t, args|
-    t.pattern = 'spec/rubocop/**/*_spec.rb'
+  desc 'Scripts and dev tools'
+  RSpec::Core::RakeTask.new(:tooling) do |t, args|
+    t.pattern = [
+      'spec/tooling/**/*_spec.rb',
+      'spec/rubocop/**/*_spec.rb'
+    ]
     t.rspec_opts = args.to_a.join(' ')
   end