diff --git a/ci/scripts/test-airgapped-build.sh b/ci/scripts/test-airgapped-build.sh index eceaec81ef720..8dbad0920ec5f 100755 --- a/ci/scripts/test-airgapped-build.sh +++ b/ci/scripts/test-airgapped-build.sh @@ -31,10 +31,10 @@ sudo ip netns exec airgapped sudo -u "$USER" \ BAZEL_BITSTREAMS_CACHE="${PWD}/bazel-airgapped/bitstreams-cache" \ OT_AIRGAPPED="true" \ BITSTREAM="--offline latest" \ - "${PWD}/bazel-airgapped/bazel" build \ - --distdir="${PWD}/bazel-airgapped/bazel-distdir" \ + "${PWD}/bazel-airgapped/bazel" build \ --vendor_dir="${PWD}/bazel-airgapped/bazel-vendor" \ --define DISABLE_VERILATOR_BUILD=true \ - //sw/device/silicon_creator/rom:mask_rom + //sw/device/silicon_creator/rom:mask_rom \ + //sw/device/tests:uart_smoketest_fpga_cw340_sival exit 0 diff --git a/rules/bitstreams.bzl b/rules/bitstreams.bzl index 4840be0a89707..3154dcb64672e 100644 --- a/rules/bitstreams.bzl +++ b/rules/bitstreams.bzl @@ -61,6 +61,10 @@ def _bitstreams_repo_impl(rctx): rctx.watch(rctx.attr.python_interpreter) rctx.watch(rctx.attr._cache_manager) + # The python binary at @python3_host//:python lives at the repo root with + # RPATH $ORIGIN/../lib/, which only works when invoked from a bin/ subdir. + # Set PYTHONHOME explicitly so Python finds its stdlib regardless. + python_home = rctx.path(rctx.attr.python_interpreter).dirname result = rctx.execute( [ rctx.path(rctx.attr.python_interpreter), @@ -71,6 +75,7 @@ def _bitstreams_repo_impl(rctx): "--cache={}".format(cache_path), "--refresh-time={}".format(rctx.attr.refresh_time), ], + environment = {"PYTHONHOME": str(python_home)}, quiet = False, ) if result.return_code != 0: diff --git a/rules/scripts/bitstreams_workspace.py b/rules/scripts/bitstreams_workspace.py index 3d699426babc1..eac1d4e56f1fe 100755 --- a/rules/scripts/bitstreams_workspace.py +++ b/rules/scripts/bitstreams_workspace.py @@ -145,8 +145,22 @@ def ResolveGitRef(self, repodir: str, ref: str): self.watch_list.append(path) - with open(path) as f: - content = f.read().strip() + try: + with open(path) as f: + content = f.read().strip() + except FileNotFoundError: + # The ref may be stored in packed-refs rather than as a loose file. + # Add packed-refs to watch list and resolve via git rev-parse. + packed_refs = subprocess.check_output( + ['git', 'rev-parse', '--path-format=absolute', '--git-path', 'packed-refs'], + universal_newlines=True, + cwd=repodir).strip() + if packed_refs not in self.watch_list: + self.watch_list.append(packed_refs) + return subprocess.check_output( + ['git', 'rev-parse', ref], + universal_newlines=True, + cwd=repodir).strip() # For refs, recurse into contents. if content.startswith('ref: '): diff --git a/util/prep-bazel-airgapped-build.sh b/util/prep-bazel-airgapped-build.sh index f453529a9273a..ce79f9bc91fe4 100755 --- a/util/prep-bazel-airgapped-build.sh +++ b/util/prep-bazel-airgapped-build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright lowRISC contributors (OpenTitan project). # Licensed under the Apache License, Version 2.0, see LICENSE for details. # SPDX-License-Identifier: Apache-2.0 @@ -10,13 +10,11 @@ set -euo pipefail : "${BAZELISK:=${REPO_TOP}/bazelisk.sh}" : "${BAZEL_VERSION:=$(cat "${REPO_TOP}/.bazelversion")}" -: "${BAZEL_AIRGAPPED_DIR:=bazel-airgapped}" -: "${BAZEL_DISTDIR:=bazel-distdir}" -: "${BAZEL_CACHEDIR:=bazel-cache}" -: "${BAZEL_VENDORDIR:=bazel-vendor}" -: "${BAZEL_BITSTREAMS_CACHE:=bitstreams-cache}" +: "${BAZEL_AIRGAPPED_DIR:=${REPO_TOP}/bazel-airgapped}" +: "${BAZEL_CACHEDIR:=${BAZEL_AIRGAPPED_DIR}/bazel-cache}" +: "${BAZEL_VENDORDIR:=${BAZEL_AIRGAPPED_DIR}/bazel-vendor}" +: "${BAZEL_BITSTREAMS_CACHE:=${BAZEL_AIRGAPPED_DIR}/bitstreams-cache}" : "${BAZEL_BITSTREAMS_CACHEDIR:=${BAZEL_BITSTREAMS_CACHE}/cache}" -: "${BAZEL_BITSTREAMS_REPO:=bitstreams}" LINE_SEP="=====================================================================" @@ -28,15 +26,15 @@ usage() { Utility script to prepare a directory with all bazel dependencies needed to build project artifacts with bazel in an airgapped environment. -Usage: $0 [-c ALL | DISTDIR | CACHE] +Usage: $0 [-c ALL | BAZEL | VENDOR] - - c: airgapped directory contents, set to either ALL or DISTDIR or CACHE. + - c: airgapped directory contents, set to either ALL, BAZEL, or VENDOR. - f: force rebuild of airgapped directory, overwriting any existing one. -Airgapped directory contents (-b): - - ALL: both the distdir and cache will be added. (Default) - - DISTDIR: only the distdir, containing bazel and its dependencies will be added. - - CACHE: only the OpenTitan bazel workspace dependencies will be added. +Airgapped directory contents (-c): + - ALL: both the bazel binary and vendor dir will be added. (Default) + - BAZEL: only the bazel binary will be added. + - VENDOR: only the OpenTitan bazel mod dependencies will be added. USAGE } @@ -70,10 +68,10 @@ if [[ "$#" -gt 0 ]]; then fi if [[ ${AIRGAPPED_DIR_CONTENTS} != "ALL" && \ - ${AIRGAPPED_DIR_CONTENTS} != "DISTDIR" && \ - ${AIRGAPPED_DIR_CONTENTS} != "CACHE" ]]; then + ${AIRGAPPED_DIR_CONTENTS} != "BAZEL" && \ + ${AIRGAPPED_DIR_CONTENTS} != "VENDOR" ]]; then echo "Invalid -c option: ${AIRGAPPED_DIR_CONTENTS}." >&2 - echo "Expected ALL, DISTDIR, or CACHE." >&2 + echo "Expected ALL, BAZEL, or VENDOR." >&2 exit 1 fi @@ -103,56 +101,55 @@ fi mkdir -p ${BAZEL_AIRGAPPED_DIR} ################################################################################ -# Prepare the distdir. +# Download bazel. ################################################################################ if [[ ${AIRGAPPED_DIR_CONTENTS} == "ALL" || \ - ${AIRGAPPED_DIR_CONTENTS} == "DISTDIR" ]]; then + ${AIRGAPPED_DIR_CONTENTS} == "BAZEL" ]]; then echo $LINE_SEP - echo "Preparing bazel offline distdir ..." - mkdir -p ${BAZEL_AIRGAPPED_DIR}/${BAZEL_DISTDIR} - cd ${BAZEL_AIRGAPPED_DIR} + echo "Downloading bazel ..." + pushd ${BAZEL_AIRGAPPED_DIR} curl --silent --location \ https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-linux-x86_64 \ --output bazel chmod +x bazel - - # Make Bazel fetch its own dependencies to the repository cache: - # https://bazel.build/run/build#repository_cache_with_bazel_7_or_later - mkdir -p "${BAZEL_AIRGAPPED_DIR}/empty_workspace" - pushd "${BAZEL_AIRGAPPED_DIR}/empty_workspace" - touch MODULE.bazel - cp "${REPO_TOP}/.bazelversion" . - "$BAZELISK" fetch --repository_cache="${BAZEL_AIRGAPPED_DIR}/${BAZEL_CACHEDIR}" popd - rm -rf "${BAZEL_AIRGAPPED_DIR}/empty_workspace" fi ################################################################################ # Prepare the cache. ################################################################################ if [[ ${AIRGAPPED_DIR_CONTENTS} == "ALL" || \ - ${AIRGAPPED_DIR_CONTENTS} == "CACHE" ]]; then + ${AIRGAPPED_DIR_CONTENTS} == "VENDOR" ]]; then echo $LINE_SEP - echo "Preparing bazel offline cachedir ..." + echo "Preparing bazel offline vendor_dir ..." cd ${REPO_TOP} - mkdir -p ${BAZEL_AIRGAPPED_DIR}/${BAZEL_CACHEDIR} + mkdir -p ${BAZEL_CACHEDIR} # Make bazel forget everything it knows, then download everything. ${BAZELISK} clean --expunge - ${BAZELISK} vendor --vendor_dir="${BAZEL_AIRGAPPED_DIR}/${BAZEL_VENDORDIR}" //... + ${BAZELISK} vendor --vendor_dir="${BAZEL_VENDORDIR}" //... # We don't need all bitstreams in the cache, we just need the latest one so # that the cache is "initialized" and "offline" mode will work correctly. - mkdir -p ${BAZEL_AIRGAPPED_DIR}/${BAZEL_BITSTREAMS_CACHEDIR} + mkdir -p ${BAZEL_BITSTREAMS_CACHEDIR} readonly SYSTEM_BITSTREAM_CACHE="${HOME}/.cache/opentitan-bitstreams" readonly SYSTEM_BITSTREAM_CACHEDIR="${SYSTEM_BITSTREAM_CACHE}/cache" - readonly LATEST_BISTREAM_HASH_FILE="${SYSTEM_BITSTREAM_CACHE}/latest.txt" - # The revision named in latest.txt is not necessarily on disk. Induce the - # cache backend to fetch the latest bitstreams. - BITSTREAM="latest" ${BAZELISK} fetch @bitstreams//... - cp "${LATEST_BISTREAM_HASH_FILE}" \ - "${BAZEL_AIRGAPPED_DIR}/${BAZEL_BITSTREAMS_CACHE}/" - LATEST_BISTREAM_HASH=$(cat "${LATEST_BISTREAM_HASH_FILE}") - cp -r "${SYSTEM_BITSTREAM_CACHEDIR}/${LATEST_BISTREAM_HASH}" \ - "${BAZEL_AIRGAPPED_DIR}/${BAZEL_BITSTREAMS_CACHEDIR}" + readonly LATEST_BITSTREAM_HASH_FILE="${SYSTEM_BITSTREAM_CACHE}/latest.txt" + # Ensure the system bitstream cache is populated. If latest.txt is missing, + # fetch it from GCP. If the bitstream files for the recorded hash are missing, + # fetch that specific revision. Skip network access when both are present. + if [[ ! -f "${LATEST_BITSTREAM_HASH_FILE}" ]]; then + BITSTREAM="latest" ${BAZELISK} fetch @bitstreams//... + else + _latest_hash=$(cat "${LATEST_BITSTREAM_HASH_FILE}") + if [[ ! -d "${SYSTEM_BITSTREAM_CACHEDIR}/${_latest_hash}" ]]; then + BITSTREAM="${_latest_hash}" ${BAZELISK} fetch @bitstreams//... + fi + unset _latest_hash + fi + cp "${LATEST_BITSTREAM_HASH_FILE}" \ + "${BAZEL_BITSTREAMS_CACHE}/" + LATEST_BITSTREAM_HASH=$(cat "${LATEST_BITSTREAM_HASH_FILE}") + cp -r "${SYSTEM_BITSTREAM_CACHEDIR}/${LATEST_BITSTREAM_HASH}" \ + "${BAZEL_BITSTREAMS_CACHEDIR}" echo "Done." fi @@ -163,5 +160,5 @@ if [[ ${AIRGAPPED_DIR_CONTENTS} == "ALL" ]]; then echo $LINE_SEP echo "To perform an airgapped build, ship the contents of ${BAZEL_AIRGAPPED_DIR} to your airgapped environment and then:" echo "" - echo "bazel build --distdir=${BAZEL_AIRGAPPED_DIR}/${BAZEL_DISTDIR} --vendor_dir=${BAZEL_AIRGAPPED_DIR}/${BAZEL_VENDORDIR}