Add support for Bzlmod (#242)

fmeum · web-flow · commit 1dbcd9167300 · 2024-03-10T08:08:19.000-04:00
* Add support for Bzlmod Requires fixing a case of a non-hermetic Python import in `dict_validation_test` that breaks with Python 3.11, with which Bazel uses `PYTHONSAFEPATH` by default. A follow-up PR will provide the publishing setup for releases to the Bazel Central Registry (BCR). * Disable layering_check for OSS-Fuzz OSS-Fuzz builds fail with: ``` ERROR: /root/.cache/bazel/_bazel_root/17db7bb59affe64429348778697da03a/external/abseil-cpp~20230802.1/absl/strings/BUILD.bazel:591:11: Compiling absl/strings/internal/cordz_functions.cc failed: undeclared inclusion(s) in rule '@@abseil-cpp~20230802.1//absl/strings:cordz_functions': this rule is missing dependency declarations for the following files included by 'absl/strings/internal/cordz_functions.cc': 'bazel-out/k8-opt-ST-b76ee6246659/bin/external/abseil-cpp~20230802.1/absl/base/atomic_hook.cppmap' 'bazel-out/k8-opt-ST-b76ee6246659/bin/external/abseil-cpp~20230802.1/absl/base/errno_saver.cppmap' 'bazel-out/k8-opt-ST-b76ee6246659/bin/external/abseil-cpp~20230802.1/absl/base/log_severity.cppmap' ``` These files are emitted by Bazel when using the `layering_check` feature. This looks like a bug or breaking change in clang. * Revert "Disable layering_check for OSS-Fuzz" This reverts commit 60a831d. * Address review comments * Update abseil and use a custom repo_name * Add subproject to .bazelignore * Use `Label` in macros * Update Bazel to 7.0.2 and ignore WORKSPACE repos in example repo * Add _repo_mapping file if supported * Apply repo mapping to local_jdk exclusion * Fix failure if there are no runfiles * Make `local_jdk` visible
diff --git a/.bazelignore b/.bazelignore
@@ -0,0 +1 @@
+examples/bzlmod
diff --git a/.bazelrc b/.bazelrc
@@ -12,10 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# rules_fuzzing is not yet compatible with Bzlmod.
-# Use the experimental version of the flag for Bazel 5 compatibility.
-common --noexperimental_enable_bzlmod
-
 # Force the use of Clang for all builds.
 build --repo_env=CC=clang
 
diff --git a/.bazelversion b/.bazelversion
@@ -1 +1 @@
-7.0.0
+7.0.2
diff --git a/.github/workflows/bazel_test.yml b/.github/workflows/bazel_test.yml
@@ -169,3 +169,23 @@ jobs:
       - name: Run regression tests on macOS
         run: |
           bazel test --verbose_failures --test_output=all //examples/...
+  bzlmod_examples_libfuzzer:
+    name: Bzlmod examples (libFuzzer)
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Run regression tests on macOS
+        run: |
+          cd examples/bzlmod
+          bazel test --verbose_failures --test_output=all --@my_rules_fuzzing//fuzzing:cc_engine=@my_rules_fuzzing//fuzzing/engines:libfuzzer //...
+  bzlmod_examples_replay:
+    name: Bzlmod examples (replay)
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Run regression tests on macOS
+        run: |
+          cd examples/bzlmod
+          bazel test --verbose_failures --test_output=all --@my_rules_fuzzing//fuzzing:cc_engine=@my_rules_fuzzing//fuzzing/engines:replay //...
diff --git a/.gitignore b/.gitignore
@@ -14,10 +14,19 @@
 
 # Bazel-generated files.
 /bazel-*
+/examples/bzlmod/bazel-*
 
 # Visual Studio Code configuration.
 /.vscode
 
 # IntelliSense configuration generated by
 # https://github.com/grailbio/bazel-compilation-database
 compile_commands.json
+
+# IntelliJ & CLion
+.clwb
+.ijwb
+
+# The Bzlmod lockfile is platform-dependent with Python and thus hard
+# to keep up-to-date in CI. It still speeds up local development.
+MODULE.bazel.lock
diff --git a/MODULE.bazel b/MODULE.bazel
@@ -0,0 +1,75 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module(
+    name = "rules_fuzzing",
+    # Automatically updated during releases by the Publish to BCR app.
+    version = "0.0.0",
+)
+
+bazel_dep(name = "abseil-cpp", version = "20240116.1", repo_name = "com_google_absl")
+bazel_dep(name = "bazel_skylib", version = "1.5.0")
+bazel_dep(name = "platforms", version = "0.0.8")
+bazel_dep(name = "rules_python", version = "0.28.0")
+bazel_dep(name = "rules_java", version = "6.5.2")
+
+bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = True, repo_name = "io_bazel_stardoc")
+bazel_dep(name = "googletest", version = "1.14.0.bcr.1", dev_dependency = True, repo_name = "com_google_googletest")
+bazel_dep(name = "re2", version = "2023-11-01", dev_dependency = True)
+
+non_module_dependencies = use_extension("//fuzzing/private:extensions.bzl", "non_module_dependencies")
+use_repo(
+    non_module_dependencies,
+    "honggfuzz",
+    "rules_fuzzing_jazzer",
+    "rules_fuzzing_jazzer_api",
+    "rules_fuzzing_oss_fuzz",
+)
+
+SUPPORTED_PYTHON_VERSIONS = [
+    "3.8",
+    "3.9",
+    "3.10",
+    "3.11",
+    "3.12",
+]
+
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+
+[
+    python.toolchain(
+        # Required to avoid an error when running as root in OSS-Fuzz.
+        ignore_root_user_error = True,
+        is_default = python_version == SUPPORTED_PYTHON_VERSIONS[-1],
+        python_version = python_version,
+    )
+    for python_version in SUPPORTED_PYTHON_VERSIONS
+]
+
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+
+[
+    pip.parse(
+        extra_pip_args = ["--require-hashes"],
+        hub_name = "rules_fuzzing_py_deps",
+        python_version = python_version,
+        requirements_lock = "//fuzzing:requirements.txt",
+    )
+    for python_version in SUPPORTED_PYTHON_VERSIONS
+]
+
+use_repo(pip, fuzzing_py_deps = "rules_fuzzing_py_deps")
+
+java_toolchains = use_extension("@rules_java//java:extensions.bzl", "toolchains")
+use_repo(java_toolchains, "local_jdk")
diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod
@@ -0,0 +1,17 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Intentionally left blank as all deps are defined in MODULE.bazel.
+# By having this file exist, Bazel will not add the repositories defined in the implicit
+# "WORKSPACE suffix" with --enable_bzlmod.
diff --git a/examples/bzlmod/.bazelrc b/examples/bzlmod/.bazelrc
@@ -0,0 +1,17 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+common --repo_env=CC=clang
+common --@my_rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer
+common --@my_rules_fuzzing//fuzzing:cc_engine_sanitizer=asan
diff --git a/examples/bzlmod/BUILD b/examples/bzlmod/BUILD
@@ -0,0 +1,27 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@my_rules_fuzzing//fuzzing:cc_defs.bzl", "cc_fuzz_test")
+load("@my_rules_fuzzing//fuzzing:java_defs.bzl", "java_fuzz_test")
+
+cc_fuzz_test(
+    name = "cc_fuzz_test",
+    srcs = ["cc_fuzz_test.cc"],
+)
+
+java_fuzz_test(
+    name = "java_fuzz_test",
+    srcs = ["JavaFuzzTest.java"],
+    target_class = "com.example.JavaFuzzTest",
+)
diff --git a/examples/bzlmod/JavaFuzzTest.java b/examples/bzlmod/JavaFuzzTest.java
@@ -0,0 +1,23 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// A plain fuzz target that does nothing.
+
+package com.example;
+
+import com.code_intelligence.jazzer.api.FuzzedDataProvider;
+
+public class JavaFuzzTest {
+  public static void fuzzerTestOneInput(FuzzedDataProvider data) {}
+}
diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel
@@ -0,0 +1,21 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# We use a custom repo_name to verify that e.g. macros do not contain hard-coded
+# references to rules_fuzzing.
+bazel_dep(name = "rules_fuzzing", repo_name = "my_rules_fuzzing")
+local_path_override(
+    module_name = "rules_fuzzing",
+    path = "../..",
+)
diff --git a/examples/bzlmod/WORKSPACE.bzlmod b/examples/bzlmod/WORKSPACE.bzlmod
@@ -0,0 +1,17 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Intentionally left blank as all deps are defined in MODULE.bazel.
+# By having this file exist, Bazel will not add the repositories defined in the implicit
+# "WORKSPACE suffix" with --enable_bzlmod.
diff --git a/examples/bzlmod/cc_fuzz_test.cc b/examples/bzlmod/cc_fuzz_test.cc
@@ -0,0 +1,22 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// A plain fuzz target that does nothing (just returns).
+
+#include <cstddef>
+#include <cstdint>
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  return 0;
+}
diff --git a/examples/java/BUILD b/examples/java/BUILD
@@ -16,7 +16,6 @@
 # serve as regression tests. Targets that are expected to crash or hang are
 # disabled in the OSS-Fuzz integration using the "no-oss-fuzz" tag.
 
-load("@rules_cc//cc:defs.bzl", "cc_binary")
 load("//fuzzing:java_defs.bzl", "java_fuzz_test")
 
 filegroup(
diff --git a/fuzzing/private/binary.bzl b/fuzzing/private/binary.bzl
@@ -34,6 +34,8 @@ Provider for storing information about a fuzz test binary.
     fields = {
         "binary_file": "The instrumented fuzz test executable.",
         "binary_runfiles": "The runfiles of the fuzz test executable.",
+        "binary_repo_mapping_manifest": "The _repo_mapping file of the fuzz " +
+                                        "test executable.",
         "corpus_dir": "The directory of the corpus files used as input seeds.",
         "dictionary_file": "The dictionary file to use in fuzzing runs.",
         "engine_info": "The `FuzzingEngineInfo` provider of the fuzzing " +
@@ -108,9 +110,11 @@ def _fuzzing_binary_impl(ctx):
     )
     if ctx.attr._instrument_binary:
         # The attribute is a list if a transition is attached.
-        binary_runfiles = ctx.attr.binary[0][DefaultInfo].default_runfiles
+        default_info = ctx.attr.binary[0][DefaultInfo]
     else:
-        binary_runfiles = ctx.attr.binary[DefaultInfo].default_runfiles
+        default_info = ctx.attr.binary[DefaultInfo]
+    binary_runfiles = default_info.default_runfiles
+    binary_repo_mapping_manifest = getattr(default_info.files_to_run, "repo_mapping_manifest")
     other_runfiles = []
     if ctx.file.corpus:
         other_runfiles.append(ctx.file.corpus)
@@ -126,6 +130,7 @@ def _fuzzing_binary_impl(ctx):
         FuzzingBinaryInfo(
             binary_file = ctx.executable.binary,
             binary_runfiles = binary_runfiles,
+            binary_repo_mapping_manifest = binary_repo_mapping_manifest,
             corpus_dir = ctx.file.corpus,
             dictionary_file = ctx.file.dictionary,
             engine_info = ctx.attr.engine[FuzzingEngineInfo],
diff --git a/fuzzing/private/extensions.bzl b/fuzzing/private/extensions.bzl
@@ -0,0 +1,22 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Internal dependencies that are not Bazel modules."""
+
+load("//fuzzing:repositories.bzl", "rules_fuzzing_dependencies")
+
+def _non_module_dependencies(_):
+    rules_fuzzing_dependencies()
+
+non_module_dependencies = module_extension(_non_module_dependencies)
diff --git a/fuzzing/private/fuzz_test.bzl b/fuzzing/private/fuzz_test.bzl
@@ -140,7 +140,7 @@ def cc_fuzz_test(
         name,
         corpus = None,
         dicts = None,
-        engine = "@rules_fuzzing//fuzzing:cc_engine",
+        engine = Label("//fuzzing:cc_engine"),
         size = None,
         tags = None,
         timeout = None,
@@ -225,7 +225,7 @@ def java_fuzz_test(
         target_class = None,
         corpus = None,
         dicts = None,
-        engine = "@rules_fuzzing//fuzzing:java_engine",
+        engine = Label("//fuzzing:java_engine"),
         size = None,
         tags = None,
         timeout = None,
@@ -329,19 +329,19 @@ def java_fuzz_test(
     jazzer_fuzz_binary(
         name = raw_binary_name,
         sanitizer = select({
-            "@rules_fuzzing//fuzzing/private:is_oss_fuzz": native_library_sanitizer,
-            "@rules_fuzzing//fuzzing/private:use_asan": "asan",
-            "@rules_fuzzing//fuzzing/private:use_ubsan": "ubsan",
+            Label("//fuzzing/private:is_oss_fuzz"): native_library_sanitizer,
+            Label("//fuzzing/private:use_asan"): "asan",
+            Label("//fuzzing/private:use_ubsan"): "ubsan",
             "//conditions:default": "none",
         }),
         sanitizer_options = select({
-            "@rules_fuzzing//fuzzing/private:is_oss_fuzz": Label("//fuzzing/private:oss_fuzz_jazzer_sanitizer_options.sh"),
+            Label("//fuzzing/private:is_oss_fuzz"): Label("//fuzzing/private:oss_fuzz_jazzer_sanitizer_options.sh"),
             "//conditions:default": Label("//fuzzing/private:local_jazzer_sanitizer_options.sh"),
         }),
         sanitizer_runtime = select({
-            "@rules_fuzzing//fuzzing/private:is_oss_fuzz": _RUNTIME_BY_NAME[native_library_sanitizer],
-            "@rules_fuzzing//fuzzing/private:use_asan": _ASAN_RUNTIME,
-            "@rules_fuzzing//fuzzing/private:use_ubsan": _UBSAN_RUNTIME,
+            Label("//fuzzing/private:is_oss_fuzz"): _RUNTIME_BY_NAME[native_library_sanitizer],
+            Label("//fuzzing/private:use_asan"): _ASAN_RUNTIME,
+            Label("//fuzzing/private:use_ubsan"): _UBSAN_RUNTIME,
             "//conditions:default": None,
         }),
         target = raw_target_name,
diff --git a/fuzzing/private/oss_fuzz/package.bzl b/fuzzing/private/oss_fuzz/package.bzl
diff --git a/fuzzing/repositories.bzl b/fuzzing/repositories.bzl
diff --git a/fuzzing/tools/dict_validation_test.py b/fuzzing/tools/dict_validation_test.py
