Skip to content

Commit 3a30d4a

Browse files
kosakkosak
committed
feat(cpp/client, py/client, csharp/client): DH-22472, DH-22473, DH-22475: Github Actions to build (cpp-client, python, csharp) on (Ubuntu 24.04, Windows)
1 parent cf4512c commit 3a30d4a

7 files changed

Lines changed: 637 additions & 0 deletions

File tree

‎.github/workflows/clients-cpp-python-ubuntu2404.yml‎

Lines changed: 258 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
name: "Build C++ and Python Clients on Ubuntu 24.04"
2+
on: [push, workflow_dispatch]
3+
permissions:
4+
packages: write
5+
6+
#===============================================================================
7+
# Background: in this workflow there are three different kinds of caching going
8+
# on:
9+
#
10+
# 1. The outputs are Github Build artifacts, attached to every action run.
11+
# These artifacts can be downloaded from e.g.
12+
# https://github.com/deephaven/deephaven-core/actions/runs/17416308139
13+
# but the specific URL depends on the build.
14+
# We use this to upload:
15+
# - The C++ install directory
16+
# - The .whl file for the static Python client
17+
# - The .whl file for the ticking Python client
18+
#
19+
# 2. The C++ install directory is cached via actions/cache, and keyed
20+
# by the hash of the cpp-client and proto directories. This allows us
21+
# to skip recompilation if these files don't change. This is an optional
22+
# optimization that might not be very significant.
23+
#
24+
# 3. Github Packages, which is a NuGet-compatible repo accessible from
25+
# https://github.com/orgs/deephaven/packages?repo_name=deephaven-core
26+
# and controlled via the NuGet API. We use this package repository to hold
27+
# the binaries for the C++ dependencies e build (grpc, arrow, etc.). It is
28+
# managed automatically for us by cmake/vcpkg/nuget. This optimization is
29+
# important because building the C++ dependencies takes significant time
30+
# (45 minutes maybe).
31+
#===============================================================================
32+
33+
env:
34+
GH_PACKAGES_USERNAME: ${{ github.repository_owner }}
35+
GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json
36+
DHINSTALL: ${{ github.workspace }}/dhinstall
37+
38+
jobs:
39+
#=============================================================================
40+
# Identify version of Deephaven Core
41+
#=============================================================================
42+
version_job:
43+
uses: ./.github/workflows/dhc-version.yml
44+
45+
#=============================================================================
46+
# Build the C++ Client
47+
#=============================================================================
48+
cpp_job:
49+
name: Build C++ client on Ubuntu or fetch from cache
50+
runs-on: ubuntu-24.04
51+
defaults:
52+
run:
53+
shell: bash
54+
55+
steps:
56+
- name: Check out this repo
57+
uses: actions/checkout@v6
58+
59+
# Try to fetch the whole installation directory (the final product of the
60+
# build) from the Github Cache. The key to this cache is derived
61+
# from the hash of the contents of the files in the cpp-client and proto
62+
# directories. If this fetch is successful, we skip over all the build
63+
# work and just copy this cached install directory to the output artifact
64+
# for this run.
65+
66+
- name: Restore the cached installation directory, or prepare a new one
67+
id: cache-cpp-install
68+
uses: actions/cache@v5
69+
with:
70+
path: ${{ env.DHINSTALL }}
71+
key: ${{ runner.os }}-cpp-client-${{ hashFiles('cpp-client/**', 'proto/**') }}
72+
73+
# [Note: skip this step if cache-cpp-install was successful]
74+
# Check out vcpkg
75+
76+
- name: Check out vcpkg
77+
id: checkout-vcpkg
78+
if: steps.cache-cpp-install.outputs.cache-hit != 'true'
79+
uses: actions/checkout@v6
80+
with:
81+
repository: microsoft/vcpkg
82+
path: vcpkg
83+
84+
# [Note: skip this step if cache-cpp-install was successful]
85+
# Bootstrap vcpkg
86+
87+
- name: Bootstrap vcpkg
88+
if: steps.cache-cpp-install.outputs.cache-hit != 'true'
89+
run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.sh
90+
91+
# [Note: skip this step if cache-cpp-install was successful]
92+
# Get mono so that we can run nuget. This is because the nuget.exe that
93+
# ships with vcpkg is an old .NET Framework binary (Windows format), and
94+
# only mono knows how to run such a thing on Linux. Some day the maintainers
95+
# will fix this so it runs with the cross-platform "dotnet".
96+
97+
- name: Get mono so that we can run nuget
98+
if: steps.cache-cpp-install.outputs.cache-hit != 'true'
99+
run: sudo apt install mono-complete
100+
101+
# [Note: skip this step if cache-cpp-install was successful]
102+
# Adapted from
103+
# https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages?pivots=linux-runner
104+
105+
# [Note: skip this step if cache-cpp-install was successful]
106+
- name: "Configure nuget to know how to download (or upload) each cached package that we need (or create)"
107+
if: steps.cache-cpp-install.outputs.cache-hit != 'true'
108+
run: |
109+
vcpkg_exe=${{ github.workspace }}/vcpkg/vcpkg
110+
nuget_dotnet=$($vcpkg_exe fetch nuget | tail -n 1)
111+
mono $nuget_dotnet \
112+
sources add \
113+
-Source "${{ env.GH_PACKAGES_FEED }}" \
114+
-StorePasswordInClearText \
115+
-Name GitHubPackages \
116+
-UserName "${{ env.GH_PACKAGES_USERNAME }}" \
117+
-Password "${{ secrets.GITHUB_TOKEN }}"
118+
mono $nuget_dotnet \
119+
setapikey "${{ secrets.GITHUB_TOKEN }}" \
120+
-Source "${{ env.GH_PACKAGES_FEED }}"
121+
122+
# [Note: skip this step if cache-cpp-install was successful]
123+
# 1. Invoke vcpkg to:
124+
# a) Try to fetch each of the dependent libraries from the nuget cache
125+
# b) If not found, build them and insert them in the nuget cache
126+
# 2. Configure cmake to prepare for the build/install step, which comes next
127+
128+
- name: build packages and configure cmake. Cached packages will be found, or stored in github's nuget repo
129+
if: steps.cache-cpp-install.outputs.cache-hit != 'true'
130+
env:
131+
VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite"
132+
run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DVCPKG_TARGET_TRIPLET=x64-linux-dynamic-release -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets
133+
134+
# [Note: skip this step if cache-cpp-install was successful]
135+
# Build our executables and install them in the configured install directory {{ env.DHINSTALL }}
136+
137+
- name: Run cmake build and install
138+
if: steps.cache-cpp-install.outputs.cache-hit != 'true'
139+
run: cmake --build cpp-client/deephaven/build --config RelWithDebInfo --target install --parallel
140+
141+
# Upload results as build artifact. Note: if cache-cpp-install was
142+
# then none of the intervening steps were executed. In that case,
143+
# this effectively just copies the cache to the build artifact.
144+
145+
- name: Upload install directory as build artifact
146+
id: upload-install-directory
147+
uses: actions/upload-artifact@v7
148+
with:
149+
name: cpp-client-install
150+
path: ${{ env.DHINSTALL }}
151+
152+
#=============================================================================
153+
# Build the Python Static Client
154+
#=============================================================================
155+
156+
python_static_job:
157+
name: Build Python static client on Ubuntu
158+
runs-on: ubuntu-24.04
159+
needs: [version_job]
160+
defaults:
161+
run:
162+
shell: bash
163+
164+
steps:
165+
- name: Check out this repo
166+
uses: actions/checkout@v6
167+
168+
- name: Install Python
169+
uses: actions/setup-python@v6
170+
with:
171+
python-version: '3.10'
172+
173+
- name: Install the 'wheel' package
174+
run: pip3 install wheel
175+
176+
- name: Install requirements-dev.txt
177+
run: |
178+
cd ./py/client
179+
pip3 install -r requirements-dev.txt
180+
181+
- name: Run setup.py
182+
env:
183+
DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }}
184+
run: |
185+
cd ./py/client
186+
python setup.py bdist_wheel
187+
188+
- name: Upload static .whl file as build artifact, to be used directly or by dependent clients like Python ticking
189+
uses: actions/upload-artifact@v7
190+
with:
191+
name: py-static-wheel
192+
path: ./py/client/dist/*.whl
193+
194+
#=============================================================================
195+
# Build the Python Ticking Client
196+
#=============================================================================
197+
198+
python_ticking_job:
199+
name: Build Python ticking client on Ubuntu
200+
runs-on: ubuntu-24.04
201+
needs: [cpp_job, python_static_job, version_job]
202+
defaults:
203+
run:
204+
shell: bash
205+
206+
steps:
207+
- name: Check out this repo
208+
uses: actions/checkout@v6
209+
210+
- name: Get the C++ installation as a build artifact
211+
uses: actions/download-artifact@v8
212+
with:
213+
name: cpp-client-install
214+
path: ${{ env.DHINSTALL }}
215+
216+
- name: Get the Python static installation as a build artifact
217+
uses: actions/download-artifact@v8
218+
with:
219+
name: py-static-wheel
220+
path: ./py/client/dist/
221+
222+
- name: Install Python
223+
uses: actions/setup-python@v6
224+
with:
225+
python-version: '3.10'
226+
227+
- name: Install the 'wheel' and 'cython' packages
228+
run: pip3 install wheel cython
229+
230+
- name: Install requirements-dev.txt
231+
run: |
232+
cd ./py/client
233+
pip3 install -r requirements-dev.txt
234+
235+
- name: Install the static client
236+
run: pip3 install --force --no-deps ./py/client/dist/*.whl
237+
238+
- name: Build cython code
239+
env:
240+
DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }}
241+
CPPFLAGS: -I${{ env.DHINSTALL }}/include
242+
LDFLAGS: -L${{ env.DHINSTALL }}/lib
243+
run: |
244+
cd ./py/client-ticking
245+
python setup.py build_ext -i
246+
247+
- name: Run setup.py
248+
env:
249+
DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }}
250+
run: |
251+
cd ./py/client-ticking
252+
python setup.py bdist_wheel
253+
254+
- name: Upload ticking .whl file as build artifact
255+
uses: actions/upload-artifact@v7
256+
with:
257+
name: py-ticking-wheel
258+
path: ./py/client-ticking/dist/*.whl

0 commit comments

Comments
 (0)