feat: Support querying by tags for GIT affected. (google#3489)

Also smoothed out some pain points in testing

Changes made:
- Edited preput to also include repo name in project field if it
contains a GIT affected field
- Added special matching rules to match package name against repo_urls,
and GIT ecosystems
- Removed footgun in the hard coded GOOGLE_CLOUD_PROJECT being oss-vdb,
so you can edit it only in the make file to change projects. Added
run-api-server-test to run api server in test instance.

Author: another-rex

.gitignore

@@ -11,3 +11,4 @@ build/lib
 **/.history/*
 */datastore-helper/
 hurl-scripts/
+temp/*

Makefile

@@ -61,7 +61,12 @@ run-website-emulator:
 run-api-server:
 	test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth login --update-adc'"; exit 1)
 	cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .
-	cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS)
+	cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS)# Run with `make run-api-server ARGS=--no-backend` to launch esp without backend.
+
+run-api-server-test:
+	test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth login --update-adc'"; exit 1)
+	cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .
+	cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb-test $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS)
 
 # TODO: API integration tests.
 all-tests: lib-tests worker-tests importer-tests alias-tests website-tests vulnfeed-tests

gcp/api/integration_tests.py

@@ -886,7 +886,7 @@ def test_get_vuln_by_alias_not_in_db(self):
         {
             'code': 5,
             'message': 'Bug not found, but the following aliases were: '
-                       'GHSA-q97m-8853-pq76'
+                       'GHSA-q97m-8853-pq76 GO-2025-3690'
         }, response.json())
 
   def test_query_batch(self):

gcp/api/server.py

@@ -1024,15 +1024,27 @@ def _is_version_affected(affected_packages,
   range.
   """
   for affected_package in affected_packages:
-    if package_name and package_name != affected_package.package.name:
-      continue
-
     if ecosystem:
       # If package ecosystem has a :, also try ignoring parts after it.
       if not is_matching_package_ecosystem(affected_package.package.ecosystem,
                                            ecosystem):
         continue
 
+    if package_name:
+      if ecosystem == 'GIT':
+        # Special case for git ecosystems to match repo urls
+        repo_url = ''
+        for rg in affected_package.ranges:
+          if rg.repo_url != '':
+            repo_url = rg.repo_url
+            break
+
+        if package_name != repo_url:
+          continue
+      else:
+        if package_name != affected_package.package.name:
+          continue
+
     if normalize:
       if any(
           osv.normalize_tag(version) == osv.normalize_tag(v)
@@ -1212,7 +1224,6 @@ def query_by_version(
     query = query.filter(osv.Bug.ecosystem == ecosystem)
     ecosystem_info = ecosystems.get(ecosystem)
 
-  ecosystem_info = ecosystems.get(ecosystem)
   is_semver = ecosystem_info and ecosystem_info.is_semver
   supports_comparing = ecosystem_info and ecosystem_info.supports_comparing
 
@@ -1450,6 +1461,12 @@ def is_matching_package_ecosystem(package_ecosystem: str,
   """Checks if the queried ecosystem matches the affected package's ecosystem,
   considering potential variations in the package's ecosystem.
   """
+
+  # Special case for GIT queries,
+  # for which osv entries will have an empty ecosystem for.
+  if ecosystem == 'GIT' and package_ecosystem == '':
+    return True
+
   return any(eco == ecosystem for eco in (
       package_ecosystem,
       ecosystems.normalize(package_ecosystem),

gcp/api/test_server.py

@@ -20,7 +20,6 @@
 
 _ESP_PORT = 8080
 _BACKEND_PORT = 8000
-_GCP_PROJECT = 'oss-vdb'
 
 
 class ServerInstance:
@@ -49,7 +48,6 @@ def start_backend(port, log_path):
   """Start backend server."""
   log_handle = open(log_path, 'w')
   env = os.environ.copy()
-  env['GOOGLE_CLOUD_PROJECT'] = _GCP_PROJECT
 
   backend_proc = subprocess.Popen(
       [sys.executable, 'server.py', f'--port={port}'],

gcp/workers/worker/testdata/UpdateTest_dont_index_too_many_git_versions.txt

@@ -5025,7 +5025,7 @@
     'is_fixed': False,
     'issue_id': None,
     'last_modified': DatetimeWithNanoseconds(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc),
-    'project': [],
+    'project': ['https://osv-test/repo/url'],
     'public': True,
     'purl': [],
     'reference_url_types': {'https://ref.com/ref': 'WEB'},
@@ -5035,6 +5035,7 @@
                           'affected',
                           'affected-01',
                           'git',
+                          'https',
                           'https://osv-test/repo/url',
                           'last',
                           'last-affected',

osv/ecosystems/_ecosystems.py

@@ -70,6 +70,7 @@
     'Linux': OrderingUnsupportedEcosystem(),
     'OSS-Fuzz': OrderingUnsupportedEcosystem(),
     'Photon OS': OrderingUnsupportedEcosystem(),
+    'GIT': OrderingUnsupportedEcosystem(),
 }
 
 # Semver-based ecosystems, should correspond to _ecosystems above.

osv/models.py

@@ -336,7 +336,7 @@ def id(self):
 
   @property
   def repo_url(self):
-    """Repo URL."""
+    """Repo URL. Assumes there is only ever one repo URL per advisory."""
     for affected_package in self.affected_packages:
       for affected_range in affected_package.ranges:
         if affected_range.repo_url:
@@ -399,6 +399,10 @@ def _pre_put_hook(self):  # pylint: disable=arguments-differ
     self.project = list({
         pkg.package.name for pkg in self.affected_packages if pkg.package.name
     })
+
+    if self.repo_url and 'GIT' in self.ecosystem:
+      self.project.append(self.repo_url)
+
     self.project.sort()
 
     ecosystems_set = {