Skip to content

Commit c1780b5

Browse files
NoureldinYosriNoureldinYosri
authored
Add steps to build with custom toolchains (#217)
fixes #155 Create a seperate job which build the python wheel for legacy systems that use glibc 2.28+ by taking full control of the C++ toolchain Also removes WORKSPACE and replaces its content by the appropiate lines in MODULE.bzl which gets us closer to the current bazel standards #60 Also updates the versions of most dependencies to latest
1 parent b8f7ca6 commit c1780b5

4 files changed

Lines changed: 222 additions & 92 deletions

File tree

‎.github/workflows/prerelease.yml‎

Lines changed: 74 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,9 +89,82 @@ jobs:
8989
name: python-wheels-${{ matrix.os }}-${{ matrix.python-version }}
9090
path: ./bazel-bin/*.whl
9191

92+
build_wheels_for_legacy_systems:
93+
runs-on: ${{ matrix.os }}
94+
needs: [create_version]
95+
strategy:
96+
fail-fast: true
97+
matrix:
98+
os: [ubuntu-22.04]
99+
python-version: ['3.10', '3.11', '3.12', '3.13']
100+
steps:
101+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4
102+
- name: Set up Python
103+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v5
104+
with:
105+
python-version: ${{ matrix.python-version }}
106+
107+
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
108+
with:
109+
name: version-file
110+
path: version
111+
112+
113+
- name: Set Python Version
114+
env:
115+
TARGET_PYTHON: ${{ matrix.python-version }}
116+
run: |
117+
echo "set version to ${TARGET_PYTHON}"
118+
python _update_bazel_py_version.py $TARGET_PYTHON
119+
120+
- name: build custom C++ toolchain
121+
env:
122+
TARGET_PYTHON: ${{ matrix.python-version }}
123+
run: |
124+
. devtools/build_with_custom_toolchain.sh
125+
126+
127+
- name: Set up Bazel
128+
uses: bazel-contrib/setup-bazel@083175551ceeceebc757ebee2127fde78840ca77 # 0.18.0
129+
with:
130+
bazelisk-cache: true
131+
disk-cache: ${{ github.workflow }}
132+
repository-cache: true
133+
134+
135+
- name: Build package
136+
env:
137+
TARGET_PYTHON: ${{ matrix.python-version }}
138+
run: |
139+
export GLIBC_VERSION=2_28
140+
echo $GLIBC_VERSION
141+
sed "s/^MANYLINUX_VERSION.*/MANYLINUX_VERSION=\"manylinux_${GLIBC_VERSION}_x86_64.manylinux2014_x86_64\"/" BUILD -i || true
142+
bazel build --define GLIBC_VERSION=$GLIBC_VERSION --define TARGET_VERSION="$(python -c "print(\"py${TARGET_PYTHON}\".replace(\".\", \"\"))")" --define VERSION="$(cat version/version.txt)" :tesseract_decoder_wheel
143+
144+
145+
- name: Test
146+
run: |
147+
pip install pytest sinter auditwheel
148+
149+
auditwheel show $(ls bazel-bin/*.whl)
150+
151+
pip install bazel-bin/*.whl
152+
153+
mv src/py/ src/_lib # rename because `py` causes pytest issues.
154+
rm src/_lib/stub_test.py # ignore this test because it uses os flags that are set by bazel
155+
rm src/_lib/tesseract_sinter_compat_test.py # ignore this test because it mixes np.int64 and int which doesn't work with older python versions
156+
157+
PYTHONPATH=src/_lib/::$PYTHONPATH pytest src/_lib/
158+
159+
160+
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
161+
with:
162+
name: python-wheels-${{ matrix.os }}-${{ matrix.python-version }}-legacy
163+
path: ./bazel-bin/*.whl
164+
92165
release-wheels:
93166
name: Publish all wheels
94-
needs: [build_wheels]
167+
needs: [build_wheels,build_wheels_for_legacy_systems]
95168
runs-on: ubuntu-24.04
96169

97170
steps:

‎MODULE.bazel‎

Lines changed: 77 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
module(name = "tesseract", version = "1.0")
22

3-
bazel_dep(name = "bazel_skylib", version = "1.7.1")
4-
bazel_dep(name = "platforms", version = "0.0.10")
5-
bazel_dep(name = "rules_python", version = "0.40.0")
6-
bazel_dep(name = "rules_cc", version = "0.0.17")
3+
bazel_dep(name = "bazel_skylib", version = "1.9.0")
4+
bazel_dep(name = "platforms", version = "1.0.0")
5+
bazel_dep(name = "rules_python", version = "1.9.0")
6+
bazel_dep(name = "rules_cc", version = "0.2.17")
77
bazel_dep(name = "pybind11_bazel", version = "2.13.6")
88

99
DEFAULT_PYTHON_VERSION = "3.13"
@@ -29,3 +29,76 @@ pip.parse(
2929
)
3030

3131
use_repo(pip, "pypi")
32+
33+
http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
34+
git_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
35+
bazel_dep(name = "zlib", version = "1.3.2")
36+
37+
HIGHS_VERSION = "1.9.0"
38+
HIGHS_SHA_256 = "dff575df08d88583c109702c7c5c75ff6e51611e6eacca8b5b3fdfba8ecc2cb4"
39+
40+
git_repository(
41+
name = "stim",
42+
commit = "bd60b73525fd5a9b30839020eb7554ad369e4337",
43+
remote = "https://github.com/quantumlib/stim.git",
44+
shallow_since = "1741128853 +0000",
45+
)
46+
47+
http_archive(
48+
name = "highs",
49+
sha256 = HIGHS_SHA_256,
50+
build_file = "//external:highs.BUILD",
51+
strip_prefix = "HiGHS-" + HIGHS_VERSION,
52+
urls = ["https://github.com/ERGO-Code/HiGHS/archive/refs/tags/v" + HIGHS_VERSION + ".tar.gz"],
53+
)
54+
55+
GTEST_VERSION = "1.13.0"
56+
57+
GTEST_SHA256 = "ad7fdba11ea011c1d925b3289cf4af2c66a352e18d4c7264392fead75e919363"
58+
59+
http_archive(
60+
name = "gtest",
61+
sha256 = GTEST_SHA256,
62+
strip_prefix = "googletest-%s" % GTEST_VERSION,
63+
urls = ["https://github.com/google/googletest/archive/refs/tags/v%s.tar.gz" % GTEST_VERSION],
64+
)
65+
66+
ARGPARSE_SHA_256 = "3e5a59ab7688dcd1f918bc92051a10564113d4f36c3bbed3ef596c25e519a062"
67+
68+
http_archive(
69+
name = "argparse",
70+
build_file = "//external:argparse.BUILD",
71+
sha256 = ARGPARSE_SHA_256,
72+
strip_prefix = "argparse-3.1",
73+
urls = ["https://github.com/p-ranav/argparse/archive/refs/tags/v3.1.zip"],
74+
)
75+
76+
git_repository(
77+
name = "nlohmann_json",
78+
commit = "9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03",
79+
remote = "https://github.com/nlohmann/json.git",
80+
shallow_since = "1701207391 +0100",
81+
)
82+
83+
84+
85+
86+
BOOST_VERSION = "1.83.0"
87+
BOOST_ARCHIVE_NAME = "boost_{}".format(BOOST_VERSION.replace(".", "_"))
88+
89+
http_archive(
90+
name = "boost",
91+
urls = [
92+
"https://archives.boost.io/release/{}/source/{}.tar.gz".format(
93+
BOOST_VERSION,
94+
BOOST_ARCHIVE_NAME,
95+
)
96+
],
97+
strip_prefix = BOOST_ARCHIVE_NAME,
98+
sha256 = "c0685b68dd44cc46574cce86c4e17c0f611b15e195be9848dfd0769a0a207628",
99+
build_file = "//external:boost.BUILD",
100+
)
101+
102+
cc_compatibility_proxy = use_extension("@rules_cc//cc:extensions.bzl", "compatibility_proxy")
103+
use_repo(cc_compatibility_proxy, "cc_compatibility_proxy")
104+

‎WORKSPACE‎

Lines changed: 0 additions & 87 deletions
This file was deleted.

‎devtools/build_with_custom_toolchain.sh‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# 1. Start a temporary background container named 'sysroot-builder'
2+
docker run -d --name sysroot-builder debian:10 sleep 3600
3+
4+
# 2. Run the installation and packaging process inside the container
5+
docker exec sysroot-builder bash -c "
6+
sed 's/deb.debian.org/archive.debian.org/g' /etc/apt/sources.list -i
7+
8+
apt-get update && apt-get upgrade -y && apt-get install -y build-essential libc6-dev symlinks apt-utils
9+
10+
# Convert absolute symlinks to relative (CRITICAL so Clang doesn't read your host OS files)
11+
symlinks -rc /lib /usr/lib /usr/include
12+
13+
# Create the staging directory
14+
mkdir -p /sysroot/usr/lib /sysroot/usr/include /sysroot/lib /sysroot/lib64
15+
16+
ldd --version
17+
18+
# Copy files while preserving symlinks (-a)
19+
cp -a /usr/include/* /sysroot/usr/include/
20+
cp -a /usr/lib/* /sysroot/usr/lib/
21+
cp -a /lib/* /sysroot/lib/ 2>/dev/null || true
22+
cp -a /lib64/* /sysroot/lib64/ 2>/dev/null || true
23+
24+
# Create a tarball inside the container
25+
cd /sysroot && tar -czf /sysroot.tar.gz .
26+
"
27+
28+
# 3. Copy the tarball from the container directly to your host machine
29+
docker cp sysroot-builder:/sysroot.tar.gz ./sysroot.tar.gz
30+
31+
# 4. Extract the contents into your project and clean up
32+
mkdir -p custom_sysroot
33+
tar -xzf sysroot.tar.gz -C custom_sysroot
34+
rm sysroot.tar.gz
35+
docker rm -f sysroot-builder
36+
ls -l -R custom_sysroot
37+
38+
# create custom_sysroot/BUILD
39+
echo """filegroup(
40+
name = \"sysroot\",
41+
srcs = glob([\"**/*\"]),
42+
visibility = [\"//visibility:public\"],
43+
)""" > custom_sysroot/BUILD
44+
45+
# update MODULE.bazel to use the custom toolchain
46+
echo """bazel_dep(name = \"toolchains_llvm\", version = \"1.6.0\")
47+
48+
llvm = use_extension(\"@toolchains_llvm//toolchain/extensions:llvm.bzl\", \"llvm\")
49+
llvm.toolchain(
50+
name = \"llvm_toolchain\",
51+
llvm_version = \"17.0.6\",
52+
)
53+
54+
llvm.sysroot(
55+
name = \"llvm_toolchain\",
56+
label = \"//custom_sysroot:sysroot\",
57+
targets = [\"linux-x86_64\"],
58+
)
59+
use_repo(llvm, \"llvm_toolchain\")
60+
61+
register_toolchains(\"@llvm_toolchain//:all\")
62+
""" >> MODULE.bazel
63+
64+
65+
sudo apt-get update
66+
sudo apt-get install -y libxml2
67+
68+
cd custom_sysroot/lib64/
69+
rm -f ld-linux-x86-64.so.2
70+
ln -s ../lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 ld-linux-x86-64.so.2
71+
cd ../..

0 commit comments

Comments
 (0)