feat: make the workers use the gitter for enumeration (google#5109)

yay the worker should be so much faster.
The tests had to be updated because the gitter is more thorough than the
legacy stuff (also, the gitter wasn't being set up correctly in the
tests anyway - fixed)

With enough arm-twisting, in a follow-up PR I can mock the gitter for
these tests, and add the test cases into the gitter tests directly.

Author: michaelkedar

gcp/workers/worker/testdata/UpdateTest_last_affected_git.txt

@@ -1,8 +1,13 @@
 {   'affected': [   {   'ranges': [   {   'events': [   {   'introduced': 'eefe8ec3f1f90d0e684890e810f3f21e8500a4cd'},
-                                                        {   'last_affected': '8d8242f545e9cec3e6d0d2e3f5bde8be1c659735'}],
+                                                        {   'last_affected': '8d8242f545e9cec3e6d0d2e3f5bde8be1c659735'},
+                                                        {   'introduced': 'febfac1940086bc1f6d3dc33fda0a1d1ba336209'}],
                                           'repo': 'https://osv-test/repo/url',
                                           'type': 'GIT'}],
-                        'versions': ['v0.1.1', 'v0.2']}],
+                        'versions': ['branch-v0.1.1',
+                                     'branch-v0.1.1-with-fix',
+                                     'branch_1_cherrypick_regress',
+                                     'v0.1.1',
+                                     'v0.2']}],
     'details': 'Blah blah blah\nBlah\n',
     'id': 'OSV-TEST-last-affected-01',
     'modified': '3000-01-01T00:00:00Z',

gcp/workers/worker/worker_test.py

@@ -185,7 +185,8 @@ def setUp(self):
     # Initialise fake source_repo.
     self.tmp_dir = tempfile.TemporaryDirectory()
     self.addCleanup(self.tmp_dir.cleanup)
-
+    osv.repos._GIT_MIRRORS[
+        'https://osv-test/repo/url'] = 'https://github.com/oliverchang/osv-test.git'
     self.mock_repo = tests.mock_repository(self)
     self.remote_source_repo_path = self.mock_repo.path
     self.mock_repo.add_file(
@@ -320,6 +321,7 @@ def test_update_limit(self):
 
     self.assertCountEqual(
         [
+            b'4c155795426727ea05575bd5904321def23c03f4',  # from cherrypicked limit event
             b'a2ba949290915d445d34d0e8e9de2e7ce38198fc',
             b'b1c95a196f22d06fcf80df8c6691cd113d8fefff',
             b'e1b045257bc5ca2a11d0476474f45ef77a0366c7',
@@ -329,10 +331,10 @@ def test_update_limit(self):
     )
 
   def test_update_no_introduced(self):
-    """Test update vulnerability with no introduced commit."""
+    """Test update vulnerability with introduced '0' commit."""
     task_runner = worker.TaskRunner(ndb_client, None, self.tmp_dir.name, None,
                                     None)
-
+    self.source_repo.detect_cherrypicks = True
     message = mock.Mock()
     message.attributes = {
         'source': 'source',
@@ -349,7 +351,6 @@ def test_update_no_introduced(self):
     affected_commits = list(osv.AffectedCommits.query())
     self.assertEqual(1, len(affected_commits))
     affected_commits = affected_commits[0]
-
     self.assertCountEqual(
         [
             b'00514d6f244f696e750a37083163992c6a50cfd3',
@@ -477,7 +478,9 @@ def test_update_pypi(self):
 
     self.assertCountEqual(
         [
+            b'4c155795426727ea05575bd5904321def23c03f4',
             b'b1c95a196f22d06fcf80df8c6691cd113d8fefff',
+            b'b9b3fd4732695b83c3068b7b6a14bb372ec31f98',  # cherrypicked fix commit
             b'eefe8ec3f1f90d0e684890e810f3f21e8500a4cd',
         ],
         [codecs.encode(commit, 'hex') for commit in affected_commits.commits],
@@ -528,7 +531,9 @@ def test_normalize_pypi(self):
 
     self.assertCountEqual(
         [
+            b'4c155795426727ea05575bd5904321def23c03f4',
             b'b1c95a196f22d06fcf80df8c6691cd113d8fefff',
+            b'b9b3fd4732695b83c3068b7b6a14bb372ec31f98',
             b'eefe8ec3f1f90d0e684890e810f3f21e8500a4cd',
         ],
         [codecs.encode(commit, 'hex') for commit in affected_commits.commits],
@@ -890,9 +895,13 @@ def test_last_affected_git(self):
 
     self.assertCountEqual(
         [
+            b'4c155795426727ea05575bd5904321def23c03f4',
             b'b1c95a196f22d06fcf80df8c6691cd113d8fefff',
+            b'b9b3fd4732695b83c3068b7b6a14bb372ec31f98',
             b'eefe8ec3f1f90d0e684890e810f3f21e8500a4cd',
             b'8d8242f545e9cec3e6d0d2e3f5bde8be1c659735',
+            b'febfac1940086bc1f6d3dc33fda0a1d1ba336209',
+            b'ff8cc32ba60ad9cbb3b23f0a82aad96ebe9ff76b',
         ],
         [codecs.encode(commit, 'hex') for commit in affected_commits.commits],
     )

osv/gitter_client.py

@@ -0,0 +1,104 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Functions for interacting with Gitter."""
+
+import requests
+
+from . import repository_pb2
+from .vulnerability_pb2 import Range
+from .repos import gitter_host, RepoInaccessibleError, _git_mirror
+
+# To regenerate the gitter repository python bindings:
+# pylint: disable=line-too-long
+# poetry run python3 -m grpc_tools.protoc --proto_path=../go/cmd/gitter/pb/repository/ --python_out=. --mypy_out=. ../go/cmd/gitter/pb/repository/repository.proto
+
+
+def get_affected_commits(
+    affected_range: Range,
+    detect_cherrypicks=True
+) -> tuple[list[str], list[bytes], list[repository_pb2.Event]]:
+  """
+    Get affected commits and tags from Gitter.
+    
+    Args:
+        affected_range: The range to get affected commits for.
+        detect_cherrypicks: Whether to detect cherrypicks.
+    
+    Returns:
+        A tuple of (versions, cherry_picked_events).
+    Raises:
+        RuntimeError: If GITTER_HOST is not set.
+        ValueError: If affected_range.type is not GIT or repo is not set.
+        RepoInaccessibleError: If Gitter reports that the repo is inaccessible.
+    """
+  if not gitter_host():
+    raise RuntimeError('GITTER_HOST not set')
+  if affected_range.type != Range.GIT:
+    raise ValueError('Range type must be GIT')
+  if not affected_range.repo:
+    raise ValueError('No repo in range')
+
+  gitter_request = repository_pb2.AffectedCommitsRequest(
+      url=_git_mirror(affected_range.repo),
+      detect_cherrypicks_fixed=detect_cherrypicks,
+      detect_cherrypicks_introduced=detect_cherrypicks,
+      detect_cherrypicks_limit=detect_cherrypicks,
+  )
+  for event in affected_range.events:
+    if event.introduced:
+      gitter_request.events.append(
+          repository_pb2.Event(
+              event_type=repository_pb2.EventType.INTRODUCED,
+              hash=event.introduced,
+          ))
+    if event.fixed:
+      gitter_request.events.append(
+          repository_pb2.Event(
+              event_type=repository_pb2.EventType.FIXED,
+              hash=event.fixed,
+          ))
+    if event.limit:
+      gitter_request.events.append(
+          repository_pb2.Event(
+              event_type=repository_pb2.EventType.LIMIT,
+              hash=event.limit,
+          ))
+    if event.last_affected:
+      gitter_request.events.append(
+          repository_pb2.Event(
+              event_type=repository_pb2.EventType.LAST_AFFECTED,
+              hash=event.last_affected,
+          ))
+
+  response = requests.post(
+      f'{gitter_host()}/affected-commits',
+      data=gitter_request.SerializeToString(),
+      headers={'Content-Type': 'application/x-protobuf'},
+      timeout=3600,
+  )
+  if response.status_code == 403:
+    raise RepoInaccessibleError(f'Gitter returned error: {response.text}')
+  if response.status_code != 200:
+    raise RuntimeError(f'Gitter returned error: {response.text}')
+
+  gitter_response = repository_pb2.AffectedCommitsResponse()
+  gitter_response.ParseFromString(response.content)
+  versions: list[str] = []
+  for tag in gitter_response.tags:
+    versions.append(tag.label)
+  commits: list[bytes] = []
+  for commit in gitter_response.commits:
+    commits.append(commit.hash)
+
+  return versions, commits, gitter_response.cherry_picked_events

osv/impact.py

@@ -29,9 +29,11 @@
 import pygit2.enums
 
 from . import ecosystems
+from . import gitter_client
 from . import repos
 from . import models
 from . import vulnerability_pb2
+from . import repository_pb2
 
 TAG_PREFIX = 'refs/tags/'
 BRANCH_PREFIX = 'refs/remotes/'
@@ -539,7 +541,8 @@ def update_affected_commits(bug_id, commits, public):
         public=public,
         page=num_pages)
     affected_commits.commits = [
-        codecs.decode(commit, 'hex') for commit in batch
+        commit if isinstance(commit, bytes) else codecs.decode(commit, 'hex')
+        for commit in batch
     ]
     to_put.append(affected_commits)
     num_pages += 1
@@ -651,13 +654,39 @@ def _analyze_git_ranges(repo_analyzer: RepoAnalyzer, checkout_path: str,
   Returns:
     A tuple of the set of new_versions and commits
   """
-  package_repo = None
-
+  if repos.gitter_host():
+    try:
+      versions, new_commits, cherries = gitter_client.get_affected_commits(
+          affected_range, detect_cherrypicks=repo_analyzer.detect_cherrypicks)
+      new_versions.update(versions)
+      commits.update(new_commits)
+      for cherry in cherries:
+        if cherry.event_type == repository_pb2.EventType.INTRODUCED:
+          new_introduced.add(cherry.hash)
+        elif cherry.event_type == repository_pb2.EventType.FIXED:
+          new_fixed.add(cherry.hash)
+      return new_versions, commits
+    except repos.RepoInaccessibleError:
+      # expected error, handled by the caller
+      raise
+    except Exception as e:
+      # logging an error and raising may be noisy.
+      logging.error(
+          'Failed to get affected commits from gitter: %s',
+          e,
+          extra={'json_fields': {
+              'repo_url': affected_range.repo
+          }},
+      )
+      raise
+
+  logging.warning('GITTER_HOST not set, falling back to legacy enumeration.')
   with tempfile.TemporaryDirectory() as package_repo_dir:
     # We'd prefer to only download the commit history, but detect_cherrypicks
     # requires having the blobs available to function correctly.
     blobless = not repo_analyzer.detect_cherrypicks
     repo_name = os.path.basename(affected_range.repo.rstrip('/')).rstrip('.git')
+    package_repo = None
     if checkout_path:
       # Repo name is <base-name>-<sha256(affected_range.repo)>.
       repo_hash = hashlib.sha256(affected_range.repo.encode()).hexdigest()

osv/repos.py

@@ -23,6 +23,19 @@
 import pygit2
 import pygit2.enums
 
+# We need to get GITTER_HOST from the environment, but we can't do it in the
+# global scope because it's not set in the test environment until after the
+# module is imported.
+_gitter_host = None
+
+
+def gitter_host():
+  global _gitter_host
+  if _gitter_host is None:
+    _gitter_host = os.getenv('GITTER_HOST', '')
+  return _gitter_host
+
+
 CLONE_TRIES = int(os.getenv('CLONE_TRIES', '3'))
 RETRY_SLEEP_SECONDS = 5
 
@@ -37,8 +50,6 @@
 FETCH_CACHE: dict[tuple, datetime.datetime] = {}
 FETCH_CACHE_SECONDS = 5 * 60  # 5 minutes
 
-GITTER_HOST = os.getenv('GITTER_HOST', '')
-
 
 class GitRemoteCallback(pygit2.RemoteCallbacks):
   """Authentication callbacks."""
@@ -131,7 +142,7 @@ def clone(git_url,
   """Perform a clone."""
   # Don't user Gitter for oss-fuzz-vulns repo because it requires auth
   logging.info('Cloning %s to %s.', git_url, checkout_dir)
-  if GITTER_HOST and git_url != 'ssh://github.com/google/oss-fuzz-vulns':
+  if gitter_host() and git_url != 'ssh://github.com/google/oss-fuzz-vulns':
     try:
       os.makedirs(checkout_dir, exist_ok=True)
       params = {'url': _git_mirror(git_url)}
@@ -140,7 +151,7 @@ def clone(git_url,
 
       # Long timeout duration (1hr) because it could be cloning a large repo
       resp = requests.get(
-          f'{GITTER_HOST}/git',
+          f'{gitter_host()}/git',
           params=params,
           stream=True,
           timeout=3600,