diff --git a/Makefile b/Makefile index c5b08f35b39..8bdb0bb8f89 100644 --- a/Makefile +++ b/Makefile @@ -25,9 +25,6 @@ worker-tests: importer-tests: cd gcp/workers/importer && ./run_tests.sh -alias-tests: - cd gcp/workers/alias && ./run_tests.sh - recoverer-tests: cd gcp/workers/recoverer && ./run_tests.sh diff --git a/gcp/workers/alias/alias_computation.py b/gcp/website/emulator_aliases.py old mode 100755 new mode 100644 similarity index 97% rename from gcp/workers/alias/alias_computation.py rename to gcp/website/emulator_aliases.py index afbbbc1eb05..1766fd76848 --- a/gcp/workers/alias/alias_computation.py +++ b/gcp/website/emulator_aliases.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +20,6 @@ import osv from osv import gcs, pubsub -import osv.logs ALIAS_GROUP_VULN_LIMIT = 32 VULN_ALIASES_LIMIT = 5 @@ -139,7 +137,7 @@ def transaction(): vuln_proto.SerializeToString(deterministic=True), type='gcs_retry') -def main(): +def run(): """Updates all alias groups in the datastore by re-computing existing AliasGroups and creating new AliasGroups for un-computed bugs.""" @@ -203,10 +201,3 @@ def main(): # For each updated vulnerability, update them in Datastore & GCS for vuln_id, alias_group in changed_vulns.items(): _update_vuln_with_group(vuln_id, alias_group) - - -if __name__ == '__main__': - _ndb_client = ndb.Client() - osv.logs.setup_gcp_logging('alias') - with _ndb_client.context(): - main() diff --git a/gcp/workers/alias/upstream_computation.py b/gcp/website/emulator_upstream.py similarity index 98% rename from gcp/workers/alias/upstream_computation.py rename to gcp/website/emulator_upstream.py index 950878c3bbc..9a06771aaf3 100644 --- a/gcp/workers/alias/upstream_computation.py +++ b/gcp/website/emulator_upstream.py @@ -23,7 +23,6 @@ from google.cloud import ndb import osv -import osv.logs from osv import gcs, pubsub @@ -196,7 +195,7 @@ def set_default(obj): target_upstream_group.put() -def main(): +def run(): """Updates all upstream groups in the datastore by re-computing existing UpstreamGroups and creating new UpstreamGroups for un-computed bugs.""" @@ -242,10 +241,3 @@ def main(): # Recompute the upstream hierarchies compute_upstream_hierarchy(group, upstream_groups) logging.info('Upstream hierarchy updated for bug: %s', group.db_id) - - -if __name__ == '__main__': - _ndb_client = ndb.Client() - osv.logs.setup_gcp_logging('upstream') - with _ndb_client.context(): - main() diff --git a/gcp/website/frontend_emulator.py b/gcp/website/frontend_emulator.py index c9bf8404080..9431afb125f 100644 --- a/gcp/website/frontend_emulator.py +++ b/gcp/website/frontend_emulator.py @@ -21,7 +21,9 @@ from google.cloud import ndb import osv import datetime -from gcp.workers.alias import upstream_computation, alias_computation + +import emulator_aliases +import emulator_upstream if __name__ == '__main__': # The datastore emulator needs to be started before main is imported @@ -115,8 +117,8 @@ def _load_dir(emulator, dir_path: str): bug.put() # Compute upstream/alias groups based on loaded bugs. - upstream_computation.main() - alias_computation.main() + emulator_aliases.run() + emulator_upstream.run() for b in osv.Bug.query(): b.put() diff --git a/gcp/workers/alias/Dockerfile b/gcp/workers/alias/Dockerfile deleted file mode 100644 index b0e072ceaef..00000000000 --- a/gcp/workers/alias/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM gcr.io/oss-vdb/worker - -COPY alias_computation.py upstream_computation.py run_cron.sh /usr/local/bin/ -RUN chmod 755 /usr/local/bin/alias_computation.py /usr/local/bin/upstream_computation.py /usr/local/bin/run_cron.sh - -ENTRYPOINT ["run_cron.sh"] diff --git a/gcp/workers/alias/alias_computation_test.py b/gcp/workers/alias/alias_computation_test.py deleted file mode 100644 index e0349797402..00000000000 --- a/gcp/workers/alias/alias_computation_test.py +++ /dev/null @@ -1,471 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Alias computation tests.""" -import datetime -import os -import unittest - -from google.cloud import ndb -from google.protobuf import timestamp_pb2 - -import osv -import alias_computation -from osv import tests - -TEST_DATA_DIR = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'testdata') - - -class AliasTest(unittest.TestCase, tests.ExpectationTest(TEST_DATA_DIR)): - """Alias tests.""" - - def _get_aliases_from_bucket(self, vuln_id): - """Get the aliases from the vulnerabilities written to the GCS bucket.""" - bucket = osv.gcs.get_osv_bucket() - pb_blob = bucket.blob(os.path.join(osv.gcs.VULN_PB_PATH, vuln_id + '.pb')) - pb = osv.vulnerability_pb2.Vulnerability.FromString( - pb_blob.download_as_bytes()) - pb_aliases = list(pb.aliases) - - return pb_aliases - - def test_basic(self): - """Tests basic case.""" - osv.AliasGroup( - bug_ids=['aaa-123', 'aaa-124'], - last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='aaa-123', - db_id='aaa-123', - aliases=['aaa-124'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='aaa-124', - db_id='aaa-124', - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - alias_computation.main() - bug_ids = osv.AliasGroup.query( - osv.AliasGroup.bug_ids == 'aaa-123').get().bug_ids - self.assertEqual(['aaa-123', 'aaa-124'], bug_ids) - - pb_aliases = self._get_aliases_from_bucket('aaa-123') - self.assertEqual(['aaa-124'], pb_aliases) - - def test_bug_reaches_limit(self): - """Tests bug reaches limit.""" - osv.Bug( - id='aaa-111', - db_id='aaa-111', - aliases=[ - 'aaa-222', 'aaa-333', 'aaa-444', 'aaa-555', 'aaa-666', 'aaa-777' - ], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - alias_computation.main() - alias_group = osv.AliasGroup.query( - osv.AliasGroup.bug_ids == 'aaa-111').get() - self.assertIsNone(alias_group) - - pb_aliases = self._get_aliases_from_bucket('aaa-111') - self.assertEqual([], pb_aliases) - - def test_update_alias_group(self): - """Tests updating an existing alias group.""" - osv.AliasGroup( - bug_ids=['bbb-123', 'bbb-234'], - last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='bbb-123', - db_id='bbb-123', - aliases=['bbb-345', 'bbb-456'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='bbb-234', - db_id='bbb-234', - aliases=['bbb-123'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='bbb-789', - db_id='bbb-789', - aliases=['bbb-456'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - alias_computation.main() - alias_group = osv.AliasGroup.query( - osv.AliasGroup.bug_ids == 'bbb-123').get() - expected_aliases = ['bbb-234', 'bbb-345', 'bbb-456', 'bbb-789'] - self.assertEqual(['bbb-123'] + expected_aliases, alias_group.bug_ids) - self.assertNotEqual( - datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - alias_group.last_modified) - - pb_aliases = self._get_aliases_from_bucket('bbb-123') - self.assertEqual(expected_aliases, pb_aliases) - - def test_create_alias_group(self): - """Tests adding a new alias group.""" - osv.Bug( - id='test-123', - db_id='test-123', - aliases=['test-124'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='test-222', - db_id='test-222', - aliases=['test-124'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - alias_computation.main() - alias_group = osv.AliasGroup.query( - osv.AliasGroup.bug_ids == 'test-123').get() - self.assertIsNotNone(alias_group) - self.assertEqual(['test-123', 'test-124', 'test-222'], alias_group.bug_ids) - - pb_aliases = self._get_aliases_from_bucket('test-123') - self.assertEqual(['test-124', 'test-222'], pb_aliases) - - def test_delete_alias_group(self): - """Tests deleting alias groups that only has one vulnerability.""" - osv.AliasGroup( - bug_ids=['ccc-123'], - last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='ccc-123', - db_id='ccc-123', - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - alias_computation.main() - alias_group = osv.AliasGroup.query( - osv.AliasGroup.bug_ids == 'ccc-123').get() - self.assertIsNone(alias_group) - - pb_aliases = self._get_aliases_from_bucket('ccc-123') - self.assertEqual([], pb_aliases) - - def test_split_alias_group(self): - """Tests split an existing alias group into two. - AliasGroup A -> B -> C -> D, remove the B -> C alias - to get AliasGroups A -> B and C -> D.""" - osv.AliasGroup( - bug_ids=['ddd-123', 'ddd-124', 'ddd-125', 'ddd-126'], - last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='ddd-123', - db_id='ddd-123', - aliases=['ddd-124'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='ddd-124', - db_id='ddd-124', - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='ddd-125', - db_id='ddd-125', - aliases=['ddd-126'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='ddd-126', - db_id='ddd-126', - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - alias_computation.main() - alias_group = osv.AliasGroup.query( - osv.AliasGroup.bug_ids == 'ddd-123').get() - self.assertIsNotNone(alias_group) - self.assertEqual(['ddd-123', 'ddd-124'], alias_group.bug_ids) - alias_group = osv.AliasGroup.query( - osv.AliasGroup.bug_ids == 'ddd-125').get() - self.assertIsNotNone(alias_group) - self.assertEqual(['ddd-125', 'ddd-126'], alias_group.bug_ids) - - pb_aliases = self._get_aliases_from_bucket('ddd-123') - self.assertEqual(['ddd-124'], pb_aliases) - pb_aliases = self._get_aliases_from_bucket('ddd-125') - self.assertEqual(['ddd-126'], pb_aliases) - - def test_allow_list(self): - """Test allow list.""" - osv.AliasAllowListEntry(bug_id='eee-111',).put() - raw_aliases = [ - 'eee-222', 'eee-333', 'eee-444', 'eee-555', 'eee-666', 'eee-777' - ] - osv.Bug( - id='eee-111', - db_id='eee-111', - aliases=raw_aliases, - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - alias_computation.main() - alias_group = osv.AliasGroup.query( - osv.AliasGroup.bug_ids == 'eee-111').get() - self.assertEqual(7, len(alias_group.bug_ids)) - - pb_aliases = self._get_aliases_from_bucket('eee-111') - self.assertEqual(raw_aliases, pb_aliases) - - def test_deny_list(self): - """Tests deny list.""" - osv.AliasGroup( - bug_ids=['fff-124', 'fff-125'], - last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.AliasDenyListEntry(bug_id='fff-123',).put() - osv.Bug( - id='fff-123', - db_id='fff-123', - aliases=['fff-124'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='fff-124', - db_id='fff-124', - aliases=['fff-125'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - alias_computation.main() - bug_ids = osv.AliasGroup.query( - osv.AliasGroup.bug_ids == 'fff-124').get().bug_ids - self.assertEqual(['fff-124', 'fff-125'], bug_ids) - - pb_aliases = self._get_aliases_from_bucket('fff-124') - self.assertEqual(['fff-125'], pb_aliases) - - def test_merge_alias_group(self): - """Tests all bugs of one alias group have been - merged to other alias group.""" - osv.AliasGroup( - bug_ids=['ggg-123', 'ggg-124'], - last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.AliasGroup( - bug_ids=['ggg-125', 'ggg-126'], - last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='ggg-123', - db_id='ggg-123', - aliases=['ggg-124', 'ggg-125', 'ggg-126'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='ggg-125', - db_id='ggg-125', - aliases=[], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - alias_computation.main() - alias_group = osv.AliasGroup.query( - osv.AliasGroup.bug_ids == 'ggg-125').fetch() - self.assertEqual(1, len(alias_group)) - self.assertEqual(['ggg-123', 'ggg-124', 'ggg-125', 'ggg-126'], - alias_group[0].bug_ids) - - pb_aliases = self._get_aliases_from_bucket('ggg-125') - self.assertEqual(['ggg-123', 'ggg-124', 'ggg-126'], pb_aliases) - - def test_partial_merge_alias_group(self): - """Tests merging some bugs of one alias group to another alias group.""" - osv.AliasGroup( - bug_ids=['hhh-123', 'hhh-124'], - last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.AliasGroup( - bug_ids=['hhh-125', 'hhh-126', 'hhh-127'], - last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='hhh-123', - db_id='hhh-123', - aliases=['hhh-124', 'hhh-125'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='hhh-126', - db_id='hhh-126', - aliases=['hhh-127'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='hhh-125', - db_id='hhh-125', - aliases=[], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='hhh-127', - db_id='hhh-127', - aliases=[], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - alias_computation.main() - alias_group = osv.AliasGroup.query( - osv.AliasGroup.bug_ids == 'hhh-125').fetch() - self.assertEqual(1, len(alias_group)) - self.assertEqual(['hhh-123', 'hhh-124', 'hhh-125'], alias_group[0].bug_ids) - alias_group = osv.AliasGroup.query( - osv.AliasGroup.bug_ids == 'hhh-127').fetch() - self.assertEqual(1, len(alias_group)) - self.assertEqual(['hhh-126', 'hhh-127'], alias_group[0].bug_ids) - - pb_aliases = self._get_aliases_from_bucket('hhh-125') - self.assertEqual(['hhh-123', 'hhh-124'], pb_aliases) - pb_aliases = self._get_aliases_from_bucket('hhh-127') - self.assertEqual(['hhh-126'], pb_aliases) - - def test_alias_group_reaches_limit(self): - """Tests a alias group reaches limit.""" - aliases = [] - for i in range(33): - aliases.append(f'iii-{i}') - - osv.Bug( - id='iii-1', - db_id='iii-1', - aliases=aliases, - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.AliasAllowListEntry(bug_id='iii-1',).put() - alias_computation.main() - alias_group = osv.AliasGroup.query(osv.AliasGroup.bug_ids == 'iii-1').get() - self.assertIsNone(alias_group) - - pb_aliases = self._get_aliases_from_bucket('iii-1') - self.assertEqual([], pb_aliases) - - def test_to_vulnerability(self): - """Tests OSV bug to vulnerability function.""" - bug = osv.Bug( - id='jjj-123', - db_id='jjj-123', - related=['jjj-111'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - timestamp=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC)) - bug.put() - alias_group_last_modified = datetime.datetime( - 2023, 10, 1, tzinfo=datetime.UTC) - osv.AliasGroup( - bug_ids=['jjj-123', 'jjj-234', 'jjj-456'], - last_modified=alias_group_last_modified, - ).put() - osv.Bug( - id='jjj-222', - db_id='jjj-222', - related=['jjj-123'], - status=1, - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - vuln = bug.to_vulnerability() - self.assertEqual(vuln.related, ['jjj-111', 'jjj-222']) - self.assertEqual(vuln.aliases, ['jjj-234', 'jjj-456']) - modified = timestamp_pb2.Timestamp() - modified.FromDatetime(alias_group_last_modified) - self.assertEqual(modified, vuln.modified) - - -def setUpModule(): - """Set up the test module.""" - # Start the emulator BEFORE creating the ndb client - unittest.enterModuleContext(tests.datastore_emulator()) - unittest.enterModuleContext(ndb.Client().context(cache_policy=False)) - - -if __name__ == '__main__': - unittest.main() diff --git a/gcp/workers/alias/build.sh b/gcp/workers/alias/build.sh deleted file mode 100755 index 7743a60f664..00000000000 --- a/gcp/workers/alias/build.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -x -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker build -t gcr.io/oss-vdb/alias-computation:$1 . && \ -docker build -t gcr.io/oss-vdb/alias-computation:latest . && \ -docker push gcr.io/oss-vdb/alias-computation:$1 && \ -docker push gcr.io/oss-vdb/alias-computation:latest diff --git a/gcp/workers/alias/run_cron.sh b/gcp/workers/alias/run_cron.sh deleted file mode 100755 index 16e2108b9bc..00000000000 --- a/gcp/workers/alias/run_cron.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -x -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -echo "Starting alias computation" -python3 /usr/local/bin/alias_computation.py -echo "Completed alias computation" - -echo "Starting upstream computation" -python3 /usr/local/bin/upstream_computation.py -echo "Completed upstream computation" \ No newline at end of file diff --git a/gcp/workers/alias/run_tests.sh b/gcp/workers/alias/run_tests.sh deleted file mode 100755 index c453dc477da..00000000000 --- a/gcp/workers/alias/run_tests.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -ex -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cd ../worker - -# Install dependencies only if not running in Cloud Build -if [ -z "$CLOUDBUILD" ]; then - poetry sync -fi -poetry run python ../alias/alias_computation_test.py -poetry run python ../alias/upstream_computation_test.py diff --git a/gcp/workers/alias/upstream_computation_test.py b/gcp/workers/alias/upstream_computation_test.py deleted file mode 100644 index bce1a3e3ccf..00000000000 --- a/gcp/workers/alias/upstream_computation_test.py +++ /dev/null @@ -1,473 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Upstream computation tests.""" -import datetime -import json -import logging -import os -import unittest - -from google.cloud import ndb - -import osv -import upstream_computation -from osv import tests - -TEST_DATA_DIR = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'testdata') - -emulator = None - - -class UpstreamTest(unittest.TestCase, tests.ExpectationTest(TEST_DATA_DIR)): - """Upstream tests.""" - - def setUp(self): - self.maxDiff = None # pylint: disable=invalid-name - emulator.reset() - osv.Bug( - id='CVE-1', - db_id='CVE-1', - status=1, - upstream_raw=[], - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='CVE-2', - db_id='CVE-2', - status=1, - upstream_raw=['CVE-1'], - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='CVE-3', - db_id='CVE-3', - status=1, - upstream_raw=['CVE-1', 'CVE-2'], - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 1, 1, tzinfo=datetime.UTC), - ).put() - - osv.Bug( - id='CVE-2023-21400', - db_id='CVE-2023-21400', - status=1, - upstream_raw=[], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2025, 1, 14, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='UBUNTU-CVE-2023-21400', - db_id='UBUNTU-CVE-2023-21400', - status=1, - upstream_raw=['CVE-2023-21400'], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2025, 1, 14, tzinfo=datetime.UTC), - ).put() - - osv.Bug( - id='UBUNTU-CVE-2023-4004', - db_id='UBUNTU-CVE-2023-4004', - status=1, - upstream_raw=['CVE-2023-4004'], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2025, 2, 10, tzinfo=datetime.UTC), - ).put() - - osv.Bug( - id='UBUNTU-CVE-2023-4015', - db_id='UBUNTU-CVE-2023-4015', - status=1, - upstream_raw=['CVE-2023-4015'], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2025, 2, 10, tzinfo=datetime.UTC), - ).put() - - osv.Bug( - id='USN-6315-1', - db_id='USN-6315-1', - status=1, - upstream_raw=[ - "CVE-2022-40982", "CVE-2023-20593", "CVE-2023-21400", - "CVE-2023-3609", "CVE-2023-3610", "CVE-2023-3611", "CVE-2023-3776", - "CVE-2023-3777", "CVE-2023-4004", "CVE-2023-4015", - "UBUNTU-CVE-2022-40982", "UBUNTU-CVE-2023-20593", - "UBUNTU-CVE-2023-21400", "UBUNTU-CVE-2023-3609", - "UBUNTU-CVE-2023-3610", "UBUNTU-CVE-2023-3611", - "UBUNTU-CVE-2023-3776", "UBUNTU-CVE-2023-3777", - "UBUNTU-CVE-2023-4004", "UBUNTU-CVE-2023-4015" - ], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2023, 8, 29, tzinfo=datetime.UTC), - ).put() - - osv.Bug( - id='USN-6325-1', - db_id='USN-6325-1', - status=1, - upstream_raw=[ - "CVE-2022-40982", "CVE-2023-20593", "CVE-2023-21400", - "CVE-2023-3609", "CVE-2023-3610", "CVE-2023-3611", "CVE-2023-3776", - "CVE-2023-3777", "CVE-2023-4004", "CVE-2023-4015", - "UBUNTU-CVE-2022-40982", "UBUNTU-CVE-2023-20593", - "UBUNTU-CVE-2023-21400", "UBUNTU-CVE-2023-3609", - "UBUNTU-CVE-2023-3610", "UBUNTU-CVE-2023-3611", - "UBUNTU-CVE-2023-3776", "UBUNTU-CVE-2023-3777", - "UBUNTU-CVE-2023-4004", "UBUNTU-CVE-2023-4015" - ], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2023, 8, 31, tzinfo=datetime.UTC), - ).put() - - osv.Bug( - id='USN-6330-1', - db_id='USN-6330-1', - status=1, - upstream_raw=[ - "CVE-2022-40982", "CVE-2023-20593", "CVE-2023-21400", - "CVE-2023-3609", "CVE-2023-3610", "CVE-2023-3611", "CVE-2023-3776", - "CVE-2023-3777", "CVE-2023-4004", "CVE-2023-4015", - "UBUNTU-CVE-2022-40982", "UBUNTU-CVE-2023-20593", - "UBUNTU-CVE-2023-21400", "UBUNTU-CVE-2023-3609", - "UBUNTU-CVE-2023-3610", "UBUNTU-CVE-2023-3611", - "UBUNTU-CVE-2023-3776", "UBUNTU-CVE-2023-3777", - "UBUNTU-CVE-2023-4004", "UBUNTU-CVE-2023-4015" - ], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2023, 8, 31, tzinfo=datetime.UTC), - ).put() - - osv.Bug( - id='USN-6332-1', - db_id='USN-6332-1', - status=1, - upstream_raw=[ - "CVE-2022-40982", "CVE-2022-4269", "CVE-2022-48502", - "CVE-2023-0597", "CVE-2023-1611", "CVE-2023-1855", "CVE-2023-1990", - "CVE-2023-2002", "CVE-2023-20593", "CVE-2023-2124", - "CVE-2023-21400", "CVE-2023-2163", "CVE-2023-2194", "CVE-2023-2235", - "CVE-2023-2269", "CVE-2023-23004", "CVE-2023-28466", - "CVE-2023-30772", "CVE-2023-3141", "CVE-2023-32248", - "CVE-2023-3268", "CVE-2023-33203", "CVE-2023-33288", - "CVE-2023-35823", "CVE-2023-35824", "CVE-2023-35828", - "CVE-2023-35829", "CVE-2023-3609", "CVE-2023-3610", "CVE-2023-3611", - "CVE-2023-3776", "CVE-2023-3777", "CVE-2023-4004", "CVE-2023-4015", - "UBUNTU-CVE-2022-40982", "UBUNTU-CVE-2022-4269", - "UBUNTU-CVE-2022-48502", "UBUNTU-CVE-2023-0597", - "UBUNTU-CVE-2023-1611", "UBUNTU-CVE-2023-1855", - "UBUNTU-CVE-2023-1990", "UBUNTU-CVE-2023-2002", - "UBUNTU-CVE-2023-20593", "UBUNTU-CVE-2023-2124", - "UBUNTU-CVE-2023-21400", "UBUNTU-CVE-2023-2163", - "UBUNTU-CVE-2023-2194", "UBUNTU-CVE-2023-2235", - "UBUNTU-CVE-2023-2269", "UBUNTU-CVE-2023-23004", - "UBUNTU-CVE-2023-28466", "UBUNTU-CVE-2023-30772", - "UBUNTU-CVE-2023-3141", "UBUNTU-CVE-2023-32248", - "UBUNTU-CVE-2023-3268", "UBUNTU-CVE-2023-33203", - "UBUNTU-CVE-2023-33288", "UBUNTU-CVE-2023-35823", - "UBUNTU-CVE-2023-35824", "UBUNTU-CVE-2023-35828", - "UBUNTU-CVE-2023-35829", "UBUNTU-CVE-2023-3609", - "UBUNTU-CVE-2023-3610", "UBUNTU-CVE-2023-3611", - "UBUNTU-CVE-2023-3776", "UBUNTU-CVE-2023-3777", - "UBUNTU-CVE-2023-4004", "UBUNTU-CVE-2023-4015" - ], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2023, 8, 31, tzinfo=datetime.UTC), - ).put() - - osv.Bug( - id='USN-6348-1', - db_id='USN-6348-1', - status=1, - upstream_raw=[ - "CVE-2022-40982", "CVE-2023-20593", "CVE-2023-21400", - "CVE-2023-3609", "CVE-2023-3610", "CVE-2023-3611", "CVE-2023-3776", - "CVE-2023-3777", "CVE-2023-4004", "CVE-2023-4015", - "UBUNTU-CVE-2022-40982", "UBUNTU-CVE-2023-20593", - "UBUNTU-CVE-2023-21400", "UBUNTU-CVE-2023-3609", - "UBUNTU-CVE-2023-3610", "UBUNTU-CVE-2023-3611", - "UBUNTU-CVE-2023-3776", "UBUNTU-CVE-2023-3777", - "UBUNTU-CVE-2023-4004", "UBUNTU-CVE-2023-4015" - ], - source='test', - public=True, - import_last_modified=datetime.datetime(2023, 9, 6, tzinfo=datetime.UTC), - ).put() - - osv.Bug( - id='USN-7234-1', - db_id='USN-7234-1', - status=1, - upstream_raw=[ - "CVE-2023-21400", "CVE-2024-40967", "CVE-2024-53103", - "CVE-2024-53141", "CVE-2024-53164", "UBUNTU-CVE-2023-21400", - "UBUNTU-CVE-2024-40967", "UBUNTU-CVE-2024-53103", - "UBUNTU-CVE-2024-53141", "UBUNTU-CVE-2024-53164" - ], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2023, 8, 31, tzinfo=datetime.UTC), - ).put() - - osv.Bug( - id='USN-7234-3', - db_id='USN-7234-3', - status=1, - upstream_raw=[ - "CVE-2023-21400", "CVE-2024-40967", "CVE-2024-53103", - "CVE-2024-53141", "CVE-2024-53164", "UBUNTU-CVE-2023-21400", - "UBUNTU-CVE-2024-40967", "UBUNTU-CVE-2024-53103", - "UBUNTU-CVE-2024-53141", "UBUNTU-CVE-2024-53164" - ], - source='test', - public=True, - import_last_modified=datetime.datetime(2025, 2, 4, tzinfo=datetime.UTC), - ).put() - - osv.Bug( - id='USN-7234-2', - db_id='USN-7234-2', - status=1, - upstream_raw=[ - "CVE-2023-21400", "CVE-2024-40967", "CVE-2024-53103", - "CVE-2024-53141", "CVE-2024-53164", "UBUNTU-CVE-2023-21400", - "UBUNTU-CVE-2024-40967", "UBUNTU-CVE-2024-53103", - "UBUNTU-CVE-2024-53141", "UBUNTU-CVE-2024-53164" - ], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2023, 8, 31, tzinfo=datetime.UTC), - ).put() - - osv.Bug( - id='DLA-3623-1', - db_id='DLA-3623-1', - status=1, - upstream_raw=[ - "CVE-2022-39189", "CVE-2022-4269", "CVE-2023-1206", "CVE-2023-1380", - "CVE-2023-2002", "CVE-2023-2007", "CVE-2023-20588", "CVE-2023-2124", - "CVE-2023-21255", "CVE-2023-21400", "CVE-2023-2269", - "CVE-2023-2898", "CVE-2023-3090", "CVE-2023-31084", "CVE-2023-3111", - "CVE-2023-3141", "CVE-2023-3212", "CVE-2023-3268", "CVE-2023-3338", - "CVE-2023-3389", "CVE-2023-34256", "CVE-2023-34319", - "CVE-2023-35788", "CVE-2023-35823", "CVE-2023-35824", - "CVE-2023-3609", "CVE-2023-3611", "CVE-2023-3772", "CVE-2023-3773", - "CVE-2023-3776", "CVE-2023-3863", "CVE-2023-4004", "CVE-2023-40283", - "CVE-2023-4132", "CVE-2023-4147", "CVE-2023-4194", "CVE-2023-4244", - "CVE-2023-4273", "CVE-2023-42753", "CVE-2023-42755", - "CVE-2023-42756", "CVE-2023-4622", "CVE-2023-4623", "CVE-2023-4921" - ], - source='test', - public=True, - import_last_modified=datetime.datetime(2024, 1, 9, tzinfo=datetime.UTC), - ).put() - - osv.Bug( - id='SUSE-SU-2023:3313-1', - db_id='SUSE-SU-2023:3313-1', - status=1, - upstream_raw=[ - "CVE-2022-40982", "CVE-2023-0459", "CVE-2023-20569", - "CVE-2023-21400", "CVE-2023-2156", "CVE-2023-2166", - "CVE-2023-31083", "CVE-2023-3268", "CVE-2023-3567", "CVE-2023-3609", - "CVE-2023-3611", "CVE-2023-3776", "CVE-2023-4004" - ], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2023, 8, 14, tzinfo=datetime.UTC), - ).put() - - def test_compute_upstream_basic(self): - """Tests basic case. - CVE-1-> CVE-2 -> CVE-3 - Upstream of CVE-3 is CVE-2 & CVE-1 - """ - - bugs_query = osv.Bug.query( - ndb.OR(osv.Bug.upstream_raw > '', osv.Bug.upstream_raw < '')) - - bugs = {bug.db_id: bug.upstream_raw for bug in bugs_query.iter()} - bug_ids = upstream_computation.compute_upstream(bugs.get('CVE-3'), bugs) - self.assertEqual(['CVE-1', 'CVE-2'], bug_ids) - - def test_compute_upstream_example(self): - """Test real world case with multiple levels""" - - bugs_query = osv.Bug.query( - ndb.OR(osv.Bug.upstream_raw > '', osv.Bug.upstream_raw < '')) - - bugs = {bug.db_id: bug.upstream_raw for bug in bugs_query.iter()} - bug_ids = upstream_computation.compute_upstream( - bugs.get('USN-7234-3'), bugs) - self.assertEqual([ - "CVE-2023-21400", "CVE-2024-40967", "CVE-2024-53103", "CVE-2024-53141", - "CVE-2024-53164", "UBUNTU-CVE-2023-21400", "UBUNTU-CVE-2024-40967", - "UBUNTU-CVE-2024-53103", "UBUNTU-CVE-2024-53141", - "UBUNTU-CVE-2024-53164" - ], bug_ids) - - def test_incomplete_compute_upstream(self): - """ Test when incomplete upstream information is given - VULN-1 -> VULN-2, VULN-3 -> VULN-4 - """ - osv.Bug( - id='VULN-1', - db_id='VULN-1', - status=1, - upstream_raw=[], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2023, 8, 14, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='VULN-2', - db_id='VULN-2', - status=1, - upstream_raw=['VULN-1'], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2023, 8, 14, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='VULN-3', - db_id='VULN-3', - status=1, - upstream_raw=['VULN-1'], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2023, 8, 14, tzinfo=datetime.UTC), - ).put() - osv.Bug( - id='VULN-4', - db_id='VULN-4', - status=1, - upstream_raw=['VULN-3'], - source='test', - public=True, - import_last_modified=datetime.datetime( - 2023, 8, 14, tzinfo=datetime.UTC), - ).put() - bugs_query = osv.Bug.query( - ndb.OR(osv.Bug.upstream_raw > '', osv.Bug.upstream_raw < '')) - bugs = {bug.db_id: bug.upstream_raw for bug in bugs_query.iter()} - bug_ids = upstream_computation.compute_upstream(bugs.get('VULN-4'), bugs) - self.assertEqual(['VULN-1', 'VULN-3'], bug_ids) - - def _get_upstreams_from_bucket(self, vuln_id): - """Get the upstreams from the vulnerabilities written to the GCS bucket.""" - bucket = osv.gcs.get_osv_bucket() - pb_blob = bucket.blob(os.path.join(osv.gcs.VULN_PB_PATH, vuln_id + '.pb')) - pb = osv.vulnerability_pb2.Vulnerability.FromString( - pb_blob.download_as_bytes()) - pb_upstream = list(pb.upstream) - - return pb_upstream - - def test_upstream_group_basic(self): - """Test the upstream group get by db_id""" - upstream_computation.main() - osv.UpstreamGroup( - db_id='CVE-3', - upstream_ids=['CVE-1', 'CVE-2'], - last_modified=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC), - ).put() - bug_ids = osv.UpstreamGroup.query( - osv.UpstreamGroup.db_id == 'CVE-3').get().upstream_ids - self.assertEqual(['CVE-1', 'CVE-2'], bug_ids) - - pb_upstream = self._get_upstreams_from_bucket('CVE-3') - self.assertEqual(['CVE-1', 'CVE-2'], pb_upstream) - - def test_upstream_group_complex(self): - """Testing more complex, realworld case""" - upstream_ids = [ - "CVE-2023-21400", "CVE-2024-40967", "CVE-2024-53103", "CVE-2024-53141", - "CVE-2024-53164", "UBUNTU-CVE-2023-21400", "UBUNTU-CVE-2024-40967", - "UBUNTU-CVE-2024-53103", "UBUNTU-CVE-2024-53141", - "UBUNTU-CVE-2024-53164" - ] - - upstream_computation.main() - bug_ids = osv.UpstreamGroup.query( - osv.UpstreamGroup.db_id == 'USN-7234-3').get().upstream_ids - - self.assertEqual(upstream_ids, bug_ids) - - pb_upstream = self._get_upstreams_from_bucket('USN-7234-3') - self.assertEqual(upstream_ids, pb_upstream) - - def test_upstream_hierarchy_computation(self): - upstream_computation.main() - bug_ids = osv.UpstreamGroup.query( - osv.UpstreamGroup.db_id == 'CVE-3').get().upstream_hierarchy - self.assertEqual('{"CVE-3": ["CVE-2"], "CVE-2": ["CVE-1"]}', bug_ids) - - def test_upstream_hierarchy_computation_complex(self): - """Testing hierarchy computation for more complex, realworld case""" - - upstream_computation.main() - - bug_ids = osv.UpstreamGroup.query( - osv.UpstreamGroup.db_id == 'USN-7234-3').get().upstream_hierarchy - bug_ids = json.loads(bug_ids) - expected = { - 'USN-7234-3': [ - 'CVE-2024-40967', 'CVE-2024-53103', 'CVE-2024-53141', - 'CVE-2024-53164', 'UBUNTU-CVE-2023-21400', 'UBUNTU-CVE-2024-40967', - 'UBUNTU-CVE-2024-53103', 'UBUNTU-CVE-2024-53141', - 'UBUNTU-CVE-2024-53164' - ], - 'UBUNTU-CVE-2023-21400': ['CVE-2023-21400'] - } - self.assertEqual(expected, bug_ids) - - -def setUpModule(): - """Set up the test module.""" - # Start the emulator BEFORE creating the ndb client - global emulator - emulator = unittest.enterModuleContext(tests.datastore_emulator()) - unittest.enterModuleContext(ndb.Client().context(cache_policy=False)) - logging.getLogger("UpstreamTest.test_compute_upstream").setLevel( - logging.DEBUG) - - -if __name__ == '__main__': - unittest.main() diff --git a/gcp/workers/cloudbuild.yaml b/gcp/workers/cloudbuild.yaml index d8b3d9ff9fe..51390bb3038 100644 --- a/gcp/workers/cloudbuild.yaml +++ b/gcp/workers/cloudbuild.yaml @@ -53,14 +53,6 @@ steps: - GITTER_PORT=8890 waitFor: ['init', 'sync'] -- name: 'gcr.io/oss-vdb/ci' - id: 'alias-tests' - dir: gcp/workers/alias - args: ['bash', '-ex', 'run_tests.sh'] - env: - - DATASTORE_EMULATOR_PORT=8002 - waitFor: ['init', 'sync'] - - name: 'gcr.io/oss-vdb/ci' id: 'recoverer-tests' dir: gcp/workers/recoverer