diff --git a/src/clusterfuzz/_internal/build_management/build_archive.py b/src/clusterfuzz/_internal/build_management/build_archive.py index c25fa8fae0..4c94fda8dc 100644 --- a/src/clusterfuzz/_internal/build_management/build_archive.py +++ b/src/clusterfuzz/_internal/build_management/build_archive.py @@ -40,6 +40,7 @@ 'afl-showmap', 'afl-tmin', 'centipede', + 'clusterfuzz_manifest.json', 'honggfuzz', 'jazzer_agent_deploy.jar', 'jazzer_driver', @@ -421,6 +422,7 @@ def _get_common_files(self) -> List[str]: return [ 'args.gn', 'llvm-symbolizer', + 'clusterfuzz_manifest.json', ] def get_target_dependencies( diff --git a/src/clusterfuzz/_internal/build_management/build_manager.py b/src/clusterfuzz/_internal/build_management/build_manager.py index 02b84e4d18..8b72711ec6 100644 --- a/src/clusterfuzz/_internal/build_management/build_manager.py +++ b/src/clusterfuzz/_internal/build_management/build_manager.py @@ -16,6 +16,7 @@ from collections import namedtuple import contextlib import datetime +import json import os import re import shutil @@ -378,6 +379,19 @@ def _pre_setup(self): environment.set_value(self.env_prefix + 'APP_PATH', '') environment.set_value(self.env_prefix + 'APP_PATH_DEBUG', '') + def _read_schema_version_from_manifest(self, build_dir: str) -> int: + """Reads archive_schema_version from clusterfuzz_manifest.json.""" + manifest_path = os.path.join(build_dir, 'clusterfuzz_manifest.json') + if not os.path.exists(manifest_path): + return 0 + try: + with open(manifest_path) as f: + manifest = json.load(f) + return int(manifest.get('archive_schema_version') or 0) + except Exception as e: + logs.warning(f'Failed to read schema version from {manifest_path}: {e}') + return 0 + def _patch_rpath(self, binary_path, instrumented_library_paths): """Patch rpaths of a binary to point to instrumented libraries""" rpaths = get_rpaths(binary_path) @@ -390,22 +404,39 @@ def _patch_rpath(self, binary_path, instrumented_library_paths): set_rpaths(binary_path, rpaths) - def _patch_rpaths(self, instrumented_library_paths): + def _patch_rpaths(self, + build_dir: str | None = None, + app_path_env: str | None = None): """Patch rpaths of builds to point to instrumented libraries.""" + instrumented_library_paths = environment.get_instrumented_libraries_paths() + if not instrumented_library_paths: + return + + if not build_dir: + build_dir = self.build_dir + + schema_version = self._read_schema_version_from_manifest(build_dir) + if schema_version > 0: + logs.info('Skipping RPATH patching for schema v1+ build.') + return + if environment.is_engine_fuzzer_job(): # Import here as this path is not available in App Engine context. from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils - for target_path in fuzzer_utils.get_fuzz_targets(self.build_dir): + for target_path in fuzzer_utils.get_fuzz_targets(build_dir): self._patch_rpath(target_path, instrumented_library_paths) else: - app_path = environment.get_value('APP_PATH') - if app_path: - self._patch_rpath(app_path, instrumented_library_paths) - - app_path_debug = environment.get_value('APP_PATH_DEBUG') - if app_path_debug: - self._patch_rpath(app_path_debug, instrumented_library_paths) + if app_path_env: + app_paths = [environment.get_value(app_path_env)] + else: + app_paths = [ + environment.get_value('APP_PATH'), + environment.get_value('APP_PATH_DEBUG') + ] + for app_path in app_paths: + if app_path: + self._patch_rpath(app_path, instrumented_library_paths) def _post_setup_success(self, update_revision=True): """Common post-setup.""" @@ -417,11 +448,6 @@ def _post_setup_success(self, update_revision=True): timestamp_file_path = os.path.join(self.base_build_dir, TIMESTAMP_FILE) utils.write_data_to_file(time.time(), timestamp_file_path) - # Update rpaths if necessary (for e.g. instrumented libraries). - instrumented_library_paths = environment.get_instrumented_libraries_paths() - if instrumented_library_paths: - self._patch_rpaths(instrumented_library_paths) - @contextlib.contextmanager def _download_and_open_build_archive(self, base_build_dir: str, build_dir: str, build_url: str): @@ -769,6 +795,7 @@ def setup(self): self._setup_application_path(build_update=build_update) self._post_setup_success(update_revision=build_update) + self._patch_rpaths() return True @@ -879,12 +906,16 @@ def setup(self): self._setup_application_path( self.release_build_dir, build_update=build_update) environment.set_value('BUILD_URL', self.release_build_url) + self._patch_rpaths( + build_dir=self.release_build_dir, app_path_env='APP_PATH') if self.debug_build_url: # Note: this will override LLVM_SYMBOLIZER_PATH, APP_DIR etc from the # previous release setup, which may not be desirable behaviour. self._setup_application_path( self.debug_build_dir, 'APP_PATH_DEBUG', build_update=build_update) + self._patch_rpaths( + build_dir=self.debug_build_dir, app_path_env='APP_PATH_DEBUG') self._post_setup_success(update_revision=build_update) return True @@ -989,6 +1020,7 @@ def setup(self): self._fuzz_targets = list(self._get_fuzz_targets_from_dir(self.build_dir)) self._setup_application_path(build_update=build_update) self._post_setup_success(update_revision=build_update) + self._patch_rpaths() return True diff --git a/src/clusterfuzz/_internal/tests/core/build_management/build_manager_test.py b/src/clusterfuzz/_internal/tests/core/build_management/build_manager_test.py index d1c3cc868e..7482461077 100644 --- a/src/clusterfuzz/_internal/tests/core/build_management/build_manager_test.py +++ b/src/clusterfuzz/_internal/tests/core/build_management/build_manager_test.py @@ -1580,6 +1580,32 @@ def test_patch_rpaths_existing_msan(self): rpaths = build_manager.get_rpaths(os.environ['APP_PATH']) self.assertListEqual(['/msan/lib', '/msan/usr/lib'], rpaths) + def test_patch_rpaths_schema_v1(self): + """Tests that RPATH is not patched when schema version is > 0.""" + + def mock_unpack_build_schema_v1(actual_self, + base_build_dir, + build_dir, + url, + http_build_url=None): + self.mock_unpack_build('rpath_new', actual_self, base_build_dir, + build_dir, url, http_build_url) + manifest_path = os.path.join(build_dir, 'clusterfuzz_manifest.json') + utils.write_data_to_file('{"archive_schema_version": 1}', manifest_path) + return True + + self.mock._unpack_build.side_effect = mock_unpack_build_schema_v1 + + build = build_manager.RegularBuild(self.base_build_dir, 1337, + 'chained-origins') + self.assertTrue(build.setup()) + self.assertEqual( + os.path.join(self.base_build_dir, 'revisions', 'app'), + os.environ['APP_PATH']) + + rpaths = build_manager.get_rpaths(os.environ['APP_PATH']) + self.assertListEqual([], rpaths) + class SortBuildUrlsByRevisionTest(unittest.TestCase): """Test _sort_build_urls_by_revision.""" @@ -1969,3 +1995,46 @@ def test_is_discovery_non_fuzzing(self): self.mock.is_engine_fuzzer_job.return_value = False build = build_manager.Build('/base', 1, build_prefix='', fuzz_target=None) self.assertFalse(build.is_discovery) + + +class ReadSchemaVersionFromManifestTest(fake_filesystem_unittest.TestCase): + """Tests _read_schema_version_from_manifest resolution.""" + + def setUp(self): + fake_filesystem_unittest.TestCase.setUp(self) + self.setUpPyfakefs() + self.base_build_dir = '/base_build_dir' + self.fs.create_dir(self.base_build_dir) + + def test_regular_build(self): + """Tests reading schema version from manifest for regular build.""" + build = build_manager.RegularBuild(self.base_build_dir, 1337, + 'gs://dummy/url.zip') + self.fs.create_file( + os.path.join(build.build_dir, 'clusterfuzz_manifest.json'), + contents='{"archive_schema_version": 1}') + self.assertEqual( + build._read_schema_version_from_manifest(build.build_dir), 1) + + def test_symbolized_build(self): + """Tests reading schema version from manifest for release and debug dirs.""" + build = build_manager.SymbolizedBuild(self.base_build_dir, 1337, + 'gs://dummy/release.zip', + 'gs://dummy/debug.zip') + self.fs.create_file( + os.path.join(build.release_build_dir, 'clusterfuzz_manifest.json'), + contents='{"archive_schema_version": 1}') + self.fs.create_file( + os.path.join(build.debug_build_dir, 'clusterfuzz_manifest.json'), + contents='{"archive_schema_version": 2}') + self.assertEqual( + build._read_schema_version_from_manifest(build.release_build_dir), 1) + self.assertEqual( + build._read_schema_version_from_manifest(build.debug_build_dir), 2) + + def test_missing_file(self): + """Tests that 0 is returned if manifest file is missing.""" + build = build_manager.RegularBuild(self.base_build_dir, 1337, + 'gs://dummy/url.zip') + self.assertEqual( + build._read_schema_version_from_manifest(build.build_dir), 0)