feat(compression): handle minimal unified diffs

Author: masterkain

lib/codex_pooler/gateway/request_compression/content_detector.ex

@@ -125,16 +125,26 @@ defmodule CodexPooler.Gateway.RequestCompression.ContentDetector do
   end
 
   defp diff_points(content) do
+    hunks = scan_count(@diff_hunk_regex, content)
+    additions = scan_count(@diff_addition_regex, content)
+    deletions = scan_count(@diff_deletion_regex, content)
+    changes = additions + deletions
+
+    if hunks == 0 or changes == 0 do
+      0
+    else
+      diff_points(content, hunks, additions, deletions, changes)
+    end
+  end
+
+  defp diff_points(content, hunks, additions, deletions, changes) do
     [
-      score(regex_match?(@diff_git_regex, content), 25),
-      score(regex_match?(@diff_index_regex, content), 10),
-      score(regex_match?(@diff_file_header_regex, content), 25),
-      score(regex_match?(@diff_hunk_regex, content), 25),
-      score(
-        regex_match?(@diff_addition_regex, content) and
-          regex_match?(@diff_deletion_regex, content),
-        20
-      )
+      cond_score(hunks, [{2, 50}, {1, 45}]),
+      cond_score(changes, [{2, 30}, {1, 25}]),
+      score(additions > 0 and deletions > 0, 10),
+      score(regex_match?(@diff_file_header_regex, content), 15),
+      score(regex_match?(@diff_git_regex, content), 10),
+      score(regex_match?(@diff_index_regex, content), 5)
     ]
     |> Enum.sum()
     |> min(100)

lib/codex_pooler/gateway/request_compression/strategies/diff.ex

@@ -10,6 +10,7 @@ defmodule CodexPooler.Gateway.RequestCompression.Strategies.Diff do
   @default_max_hunks 32
   @default_max_hunks_per_file 8
   @default_context_lines 2
+  @default_model "gpt-4o"
 
   @file_start_regex ~r/^diff --git\s+\S+\s+\S+/
   @hunk_header_regex ~r/^@@\s+-\d+(?:,\d+)?\s+\+\d+(?:,\d+)?\s+@@/
@@ -36,7 +37,7 @@ defmodule CodexPooler.Gateway.RequestCompression.Strategies.Diff do
 
         compressed = Strategies.join_lines(compressed_lines)
 
-        Strategies.finalize(
+        finalize(
           @strategy,
           content,
           compressed,
@@ -198,7 +199,14 @@ defmodule CodexPooler.Gateway.RequestCompression.Strategies.Diff do
         not changed_line?(line) and index in selected_indexes
       end)
 
-    {[hunk.header | body_lines], kept_context_line_count, omitted_context_line_count}
+    hunk_lines =
+      if kept_context_line_count == 0 and omitted_context_line_count == 0 do
+        body_lines
+      else
+        [hunk.header | body_lines]
+      end
+
+    {hunk_lines, kept_context_line_count, omitted_context_line_count}
   end
 
   defp selected_hunk_indexes(lines, context_lines) do
@@ -281,4 +289,20 @@ defmodule CodexPooler.Gateway.RequestCompression.Strategies.Diff do
     (String.starts_with?(line, "+") and not String.starts_with?(line, "+++")) or
       (String.starts_with?(line, "-") and not String.starts_with?(line, "---"))
   end
+
+  defp finalize(strategy, original, compressed, counts, opts) do
+    Strategies.finalize(strategy, original, compressed, counts, default_model_opts(opts))
+  end
+
+  defp default_model_opts(opts) when is_list(opts) do
+    if Keyword.has_key?(opts, :model), do: opts, else: Keyword.put(opts, :model, @default_model)
+  end
+
+  defp default_model_opts(opts) when is_map(opts) do
+    if Map.has_key?(opts, :model) or Map.has_key?(opts, "model") do
+      opts
+    else
+      Map.put(opts, :model, @default_model)
+    end
+  end
 end

test/codex_pooler/gateway/request_compression/content_detector_test.exs

@@ -100,6 +100,53 @@ defmodule CodexPooler.Gateway.RequestCompression.ContentDetectorTest do
       assert_noop(:text, ContentDetector.detect(negative))
     end
 
+    test "detects one-sided and minimal unified hunks without diff git headers" do
+      additions_only = """
+      @@ -1,0 +1,3 @@
+      +added one
+      +added two
+      +added three
+      """
+
+      deletions_only = """
+      @@ -1,3 +1,0 @@
+      -removed one
+      -removed two
+      -removed three
+      """
+
+      replacement = """
+      @@ -1 +1 @@
+      -old value
+      +new value
+      """
+
+      for body <- [additions_only, deletions_only, replacement] do
+        assert %{kind: :diff, confidence: confidence, compressible: true, strategy: :diff} =
+                 ContentDetector.detect(body)
+
+        assert confidence >= 0.7
+      end
+    end
+
+    test "keeps prose with isolated plus and minus lines out of diff detection" do
+      prose = """
+      Review notes for the next change:
+      + add a short summary before the examples
+      - remove the stale paragraph near the end
+      """
+
+      missing_hunk = """
+      --- a/example.txt
+      +++ b/example.txt
+      +added without a hunk header
+      -removed without a hunk header
+      """
+
+      assert_noop(:text, ContentDetector.detect(prose))
+      assert_noop(:text, ContentDetector.detect(missing_hunk))
+    end
+
     test "detects HTML at request-compression confidence but keeps it no-op" do
       positive = """
       <!doctype html>

test/codex_pooler/gateway/request_compression/strategies/diff_test.exs

@@ -99,6 +99,56 @@ defmodule CodexPooler.Gateway.RequestCompression.Strategies.DiffTest do
       assert :skip = Diff.compress("ordinary output without diff hunks", min_bytes: 0)
     end
 
+    test "compresses one-sided and replacement hunks without diff git headers" do
+      for {content, expected_additions, expected_deletions} <- [
+            {additions_only_fixture(), 1, 0},
+            {deletions_only_fixture(), 0, 1},
+            {replacement_fixture(), 1, 1}
+          ] do
+        assert {:ok, %{content: compressed, metadata: metadata}} =
+                 Diff.compress(content,
+                   model: @model,
+                   min_bytes: 0,
+                   min_hunks: 1,
+                   context_lines: 0
+                 )
+
+        assert compressed =~ "@@"
+        assert compressed =~ "[compressed diff output: omitted"
+        assert metadata.strategy == :diff
+        assert metadata.original_hunk_count == 1
+        assert metadata.compressed_hunk_count == 1
+        assert metadata.addition_line_count == expected_additions
+        assert metadata.deletion_line_count == expected_deletions
+      end
+    end
+
+    test "compresses minimal bare unified hunks and skips prose plus/minus lines" do
+      minimal = """
+      @@ -1,0 +1,2 @@
+      +one
+      +two
+      """
+
+      prose = """
+      Review notes for the next change:
+      + add a short summary before the examples
+      - remove the stale paragraph near the end
+      """
+
+      assert {:ok, %{content: compressed, metadata: metadata}} =
+               Diff.compress(minimal, min_bytes: 0, min_hunks: 1)
+
+      assert compressed == "+one\n+two"
+      assert metadata.strategy == :diff
+      assert metadata.original_hunk_count == 1
+      assert metadata.compressed_hunk_count == 1
+      assert metadata.addition_line_count == 2
+      assert metadata.deletion_line_count == 0
+
+      assert :skip = Diff.compress(prose, min_bytes: 0, min_hunks: 1)
+    end
+
     test "does not retain state between calls" do
       assert {:ok, _result} =
                Diff.compress(diff_fixture("DROP_ME_STALE_DIFF"),
@@ -151,6 +201,33 @@ defmodule CodexPooler.Gateway.RequestCompression.Strategies.DiffTest do
     """
   end
 
+  defp additions_only_fixture do
+    diff_with_context("+added value")
+  end
+
+  defp deletions_only_fixture do
+    diff_with_context("-removed value")
+  end
+
+  defp replacement_fixture do
+    diff_with_context("""
+    -old value
+    +new value
+    """)
+  end
+
+  defp diff_with_context(change) do
+    context =
+      1..24
+      |> Enum.map_join("\n", &" context line #{&1}")
+
+    """
+    @@ -1,25 +1,25 @@
+    #{context}
+    #{String.trim_trailing(change)}
+    """
+  end
+
   defp token_count(content) do
     assert {:ok, count, _metadata} = TokenCounter.count(@model, content)
     count