Skip to content

Commit 53fcdd7

Browse files
authored
Stabilize cross-platform Bazel builds (macOS + Linux) and add conda CI (#55)
* fix(workspace): declare zlib and protobuf before TensorFlow workspaces * fix(bzl): use native.cc_binary and native.cc_library in macros * fix(macos): add script and build targets to patch wrapped_clang for macOS * build: set Bazel 7.4.1 on macOS and 6.5.0 on Linux via configure.sh * ci: add GitHub Actions build workflow with conda environment for Linux and macOS
1 parent 27bb99c commit 53fcdd7

9 files changed

Lines changed: 298 additions & 15 deletions

File tree

.bazelversion

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +0,0 @@
1-
6.5.0

.github/workflows/build.yml

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: Build
2+
3+
on:
4+
push:
5+
branches: [ master ]
6+
pull_request:
7+
branches: [ master ]
8+
9+
jobs:
10+
build:
11+
name: Build (${{ matrix.os }})
12+
runs-on: ${{ matrix.os }}
13+
strategy:
14+
matrix:
15+
os: [ubuntu-latest, macos-latest]
16+
17+
steps:
18+
- uses: actions/checkout@v4
19+
20+
- name: Set up conda environment
21+
uses: conda-incubator/setup-miniconda@v3
22+
with:
23+
auto-activate-base: false
24+
activate-environment: s2t-env
25+
environment-file: environment.yml
26+
27+
- name: Install Bazel
28+
shell: bash -l {0}
29+
run: |
30+
# Install Bazelisk (manages Bazel versions)
31+
if [ "$RUNNER_OS" == "Linux" ]; then
32+
curl -Lo /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-linux-amd64
33+
elif [ "$RUNNER_OS" == "macOS" ]; then
34+
curl -Lo /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-darwin-amd64
35+
fi
36+
chmod +x /tmp/bazelisk
37+
sudo mv /tmp/bazelisk /usr/local/bin/bazel
38+
39+
- name: Configure Bazel
40+
shell: bash -l {0}
41+
run: |
42+
./configure.sh
43+
bazel --version
44+
45+
- name: Patch Apple CC toolchain (macOS only)
46+
if: runner.os == 'macOS'
47+
shell: bash -l {0}
48+
run: bazel build //:patch_local_config_apple_cc
49+
50+
- name: Build struct2tensor
51+
shell: bash -l {0}
52+
run: bazel build //struct2tensor:path --verbose_failures
53+
54+
- name: Build all targets
55+
if: runner.os == 'Linux'
56+
shell: bash -l {0}
57+
run: bazel build //...

BUILD

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,20 @@ licenses(["notice"]) # Apache 2.0
1818

1919
package(default_visibility = ["//visibility:public"])
2020

21+
genrule(
22+
name = "patch_local_config_apple_cc",
23+
tools = ["//struct2tensor/tools:patch_wrapped_clang"],
24+
outs = ["patch_local_config_apple_cc.stamp"],
25+
cmd = "$(execpath //struct2tensor/tools:patch_wrapped_clang) && touch $@",
26+
tags = [
27+
"local",
28+
"no-cache",
29+
"no-remote",
30+
"no-sandbox",
31+
],
32+
target_compatible_with = ["@platforms//os:osx"],
33+
)
34+
2135
sh_binary(
2236
name = "build_pip_package",
2337
testonly = True, # Some files are testonly

WORKSPACE

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,25 @@ tf_configure(name = "local_config_tf")
2626

2727
#####################################################################################
2828

29+
http_archive(
30+
name = "zlib",
31+
build_file = "@com_google_protobuf//:third_party/zlib.BUILD",
32+
sha256 = "17e88863f3600672ab49182f217281b6fc4d3c762bde361935e436a95214d05c",
33+
strip_prefix = "zlib-1.3.1",
34+
urls = ["https://github.com/madler/zlib/archive/v1.3.1.tar.gz"],
35+
)
36+
37+
# ===== Protobuf 4.25.6 dependency =====
38+
# Must be declared BEFORE TensorFlow's workspaces to override the version they pull
39+
http_archive(
40+
name = "com_google_protobuf",
41+
sha256 = "4e6727bc5d23177edefa3ad86fd2f5a92cd324151636212fd1f7f13aef3fd2b7",
42+
strip_prefix = "protobuf-4.25.6",
43+
urls = [
44+
"https://github.com/protocolbuffers/protobuf/archive/v4.25.6.tar.gz",
45+
],
46+
)
47+
2948
# ===== TensorFlow dependency =====
3049
#
3150
# TensorFlow is imported here instead of in tf_serving_workspace() because
@@ -64,17 +83,6 @@ local_python_configure(name = "local_execution_config_python")
6483
load("//struct2tensor:workspace.bzl", "struct2tensor_workspace")
6584
struct2tensor_workspace()
6685

67-
# ===== Protobuf 4.25.6 dependency =====
68-
# Must be declared BEFORE TensorFlow's workspaces to override the version they pull
69-
http_archive(
70-
name = "com_google_protobuf",
71-
sha256 = "4e6727bc5d23177edefa3ad86fd2f5a92cd324151636212fd1f7f13aef3fd2b7",
72-
strip_prefix = "protobuf-4.25.6",
73-
urls = [
74-
"https://github.com/protocolbuffers/protobuf/archive/v4.25.6.tar.gz",
75-
],
76-
)
77-
7886
# Load Protobuf dependencies
7987
load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
8088
protobuf_deps()

configure.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,13 @@ else
5353
fi
5454
fi
5555

56+
# Set Bazel version based on OS
57+
if [[ "$(uname)" == "Darwin" ]]; then
58+
echo "7.4.1" > .bazelversion
59+
else
60+
echo "6.5.0" > .bazelversion
61+
fi
62+
5663
ensure_tensorflow
5764
TF_CFLAGS=( $(${PYTHON_BIN_PATH} -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') )
5865
TF_LFLAGS="$(${PYTHON_BIN_PATH} -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))')"

environment.yml

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
name: s2t-env-py311
2+
channels:
3+
- conda-forge
4+
dependencies:
5+
- python=3.11
6+
- pip
7+
- pip:
8+
- absl-py==2.4.0
9+
- altgraph==0.17.5
10+
- astunparse==1.6.3
11+
- certifi==2026.2.25
12+
- charset-normalizer==3.4.5
13+
- delocate==0.13.0
14+
- flatbuffers==25.12.19
15+
- gast==0.7.0
16+
- google-pasta==0.2.0
17+
- googleapis-common-protos==1.72.0
18+
- grpcio==1.78.0
19+
- h5py==3.16.0
20+
- idna==3.11
21+
- keras==3.13.2
22+
- libclang==18.1.1
23+
- macholib==1.16.4
24+
- markdown==3.10.2
25+
- markdown-it-py==4.0.0
26+
- markupsafe==3.0.3
27+
- mdurl==0.1.2
28+
- ml-dtypes==0.4.1
29+
- namex==0.1.0
30+
- numpy==1.26.4
31+
- opt-einsum==3.4.0
32+
- optree==0.19.0
33+
- protobuf==4.25.8
34+
- pyarrow==10.0.1
35+
- pygments==2.19.2
36+
- requests==2.32.5
37+
- rich==14.3.3
38+
- six==1.17.0
39+
- tensorboard==2.17.1
40+
- tensorboard-data-server==0.7.2
41+
- tensorflow==2.17.1
42+
- tensorflow-io-gcs-filesystem==0.37.1
43+
- tensorflow-metadata==1.17.3
44+
- termcolor==3.3.0
45+
- typing-extensions==4.15.0
46+
- urllib3==2.6.3
47+
- werkzeug==3.1.6
48+
- wrapt==2.1.2

struct2tensor/struct2tensor.bzl

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ DYNAMIC_DEPS = ["@local_config_tf//:libtensorflow_framework", "@local_config_tf/
287287

288288
def s2t_dynamic_binary(name, deps):
289289
"""Creates a .so file intended for linking with tensorflow_framework.so."""
290-
cc_binary(
290+
native.cc_binary(
291291
name = name,
292292
copts = DYNAMIC_COPTS,
293293
linkshared = 1,
@@ -300,7 +300,7 @@ def s2t_dynamic_library(
300300
deps = None):
301301
"""Creates a static library intended for linking with tensorflow_framework.so."""
302302
true_deps = [] if deps == None else deps
303-
cc_library(
303+
native.cc_library(
304304
name = name,
305305
srcs = srcs,
306306
alwayslink = 1,
@@ -413,4 +413,4 @@ def py_proto_library(name, deps, visibility = None, **kwargs):
413413
deps = deps,
414414
visibility = visibility,
415415
**kwargs
416-
)
416+
)

struct2tensor/tools/BUILD

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@ package(
2121
default_visibility = ["//struct2tensor:__subpackages__"],
2222
)
2323

24+
sh_binary(
25+
name = "patch_wrapped_clang",
26+
srcs = ["patch_wrapped_clang.sh"],
27+
visibility = ["//visibility:public"],
28+
)
29+
2430
py_library(
2531
name = "build_docs_lib",
2632
srcs = ["build_docs.py"],
Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
5+
6+
find_wrapper_dir() {
7+
local -a candidates=()
8+
local cwd="$(pwd)"
9+
10+
candidates+=("${cwd}/external/local_config_apple_cc")
11+
candidates+=("${cwd}/../external/local_config_apple_cc")
12+
13+
candidates+=("${ROOT_DIR}/bazel-struct2tensor/external/local_config_apple_cc")
14+
15+
if command -v bazel >/dev/null 2>&1; then
16+
local output_base
17+
if output_base="$(cd "${ROOT_DIR}" && bazel info output_base 2>/dev/null)"; then
18+
candidates+=("${output_base}/external/local_config_apple_cc")
19+
fi
20+
fi
21+
22+
for dir in "${candidates[@]}"; do
23+
if [[ -d "${dir}" ]]; then
24+
echo "${dir}"
25+
return 0
26+
fi
27+
done
28+
29+
return 1
30+
}
31+
32+
write_wrapper() {
33+
local path="$1"
34+
cat >"${path}" <<EOF
35+
#!/bin/bash
36+
set -euo pipefail
37+
WRAPPER_DEVDIR="\${DEVELOPER_DIR:-\$(xcode-select -p)}"
38+
SDKROOT_PATH="\${SDKROOT:-\$(xcrun --sdk macosx --show-sdk-path)}"
39+
tool="clang"
40+
if [[ "\$(basename "\$0")" == "wrapped_clang_pp" ]]; then
41+
tool="clang++"
42+
fi
43+
args=()
44+
for arg in "\$@"; do
45+
arg="\${arg//__BAZEL_XCODE_DEVELOPER_DIR__/\${WRAPPER_DEVDIR}}"
46+
arg="\${arg//__BAZEL_XCODE_SDKROOT__/\${SDKROOT_PATH}}"
47+
args+=("\$arg")
48+
done
49+
exec /usr/bin/xcrun "\${tool}" "\${args[@]}"
50+
EOF
51+
chmod +x "${path}"
52+
}
53+
54+
write_libtool_check_unique() {
55+
local path="$1"
56+
cat >"${path}" <<'EOF'
57+
#!/bin/bash
58+
set -euo pipefail
59+
60+
TMP_INPUTS="$(mktemp "${TMPDIR:-/tmp}/libtool_unique.XXXXXX")"
61+
trap 'rm -f "$TMP_INPUTS"' EXIT
62+
63+
EXPECT_FILELIST=0
64+
65+
add_object() {
66+
local obj="$1"
67+
[[ -n "$obj" ]] || return 0
68+
basename "$obj" >>"$TMP_INPUTS"
69+
}
70+
71+
parse_token() {
72+
local token="$1"
73+
74+
if [[ "$EXPECT_FILELIST" == "1" ]]; then
75+
EXPECT_FILELIST=0
76+
if [[ -f "$token" ]]; then
77+
while IFS= read -r obj; do
78+
add_object "$obj"
79+
done <"$token"
80+
fi
81+
return 0
82+
fi
83+
84+
case "$token" in
85+
-filelist)
86+
EXPECT_FILELIST=1
87+
;;
88+
@*)
89+
local params_file="${token:1}"
90+
if [[ -f "$params_file" ]]; then
91+
while IFS= read -r opt; do
92+
parse_token "$opt"
93+
done <"$params_file"
94+
fi
95+
;;
96+
*.o)
97+
add_object "$token"
98+
;;
99+
esac
100+
}
101+
102+
for arg in "$@"; do
103+
parse_token "$arg"
104+
done
105+
106+
if sort "$TMP_INPUTS" | uniq -d | grep -q .; then
107+
exit 1
108+
fi
109+
110+
exit 0
111+
EOF
112+
chmod +x "${path}"
113+
}
114+
115+
main() {
116+
local wrapper_dir
117+
if ! wrapper_dir="$(find_wrapper_dir)"; then
118+
echo "Could not find local_config_apple_cc wrapper directory." >&2
119+
echo "Run a Bazel build once, then rerun this script." >&2
120+
exit 1
121+
fi
122+
123+
local clang_wrapper="${wrapper_dir}/wrapped_clang"
124+
local clangpp_wrapper="${wrapper_dir}/wrapped_clang_pp"
125+
local libtool_check_unique="${wrapper_dir}/libtool_check_unique"
126+
127+
if [[ ! -e "${clang_wrapper}" || ! -e "${clangpp_wrapper}" || ! -e "${libtool_check_unique}" ]]; then
128+
echo "Missing wrapped_clang binaries under ${wrapper_dir}" >&2
129+
exit 1
130+
fi
131+
132+
cp -f "${clang_wrapper}" "${clang_wrapper}.bak"
133+
cp -f "${clangpp_wrapper}" "${clangpp_wrapper}.bak"
134+
cp -f "${libtool_check_unique}" "${libtool_check_unique}.bak"
135+
136+
write_wrapper "${clang_wrapper}"
137+
write_wrapper "${clangpp_wrapper}"
138+
write_libtool_check_unique "${libtool_check_unique}"
139+
140+
echo "Patched wrappers in: ${wrapper_dir}"
141+
echo "Backups saved as wrapped_clang.bak, wrapped_clang_pp.bak, libtool_check_unique.bak"
142+
}
143+
144+
main "$@"

0 commit comments

Comments
 (0)