Skip to content

Commit ac939fc

Browse files
kosakkosak
committed
feat(cpp/client, py/client, csharp/client): DH-22472, DH-22473, DH-22475: Github Actions to build (cpp-client, python, csharp) on (Ubuntu 24.04, Windows)
1 parent cf4512c commit ac939fc

7 files changed

Lines changed: 649 additions & 0 deletions

File tree

‎.github/workflows/clients-cpp-python-ubuntu2404.yml‎

Lines changed: 262 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
name: "Build C++ and Python Clients on Ubuntu 24.04"
2+
on:
3+
pull_request:
4+
branches: [ 'main', 'rc/v*' ]
5+
push:
6+
branches: [ 'main', 'check/**', 'release/v*' ]
7+
permissions:
8+
packages: write
9+
10+
#===============================================================================
11+
# Background: in this workflow there are three different kinds of caching going
12+
# on:
13+
#
14+
# 1. The outputs are Github Build artifacts, attached to every action run.
15+
# These artifacts can be downloaded from e.g.
16+
# https://github.com/deephaven/deephaven-core/actions/runs/17416308139
17+
# but the specific URL depends on the build.
18+
# We use this to upload:
19+
# - The C++ install directory
20+
# - The .whl file for the static Python client
21+
# - The .whl file for the ticking Python client
22+
#
23+
# 2. The C++ install directory is cached via actions/cache, and keyed
24+
# by the hash of the cpp-client and proto directories. This allows us
25+
# to skip recompilation if these files don't change. This is an optional
26+
# optimization that might not be very significant.
27+
#
28+
# 3. Github Packages, which is a NuGet-compatible repo accessible from
29+
# https://github.com/orgs/deephaven/packages?repo_name=deephaven-core
30+
# and controlled via the NuGet API. We use this package repository to hold
31+
# the binaries for the C++ dependencies e build (grpc, arrow, etc.). It is
32+
# managed automatically for us by cmake/vcpkg/nuget. This optimization is
33+
# important because building the C++ dependencies takes significant time
34+
# (45 minutes maybe).
35+
#===============================================================================
36+
37+
env:
38+
GH_PACKAGES_USERNAME: ${{ github.repository_owner }}
39+
GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json
40+
DHINSTALL: ${{ github.workspace }}/dhinstall
41+
42+
jobs:
43+
#=============================================================================
44+
# Identify version of Deephaven Core
45+
#=============================================================================
46+
version_job:
47+
uses: ./.github/workflows/dhc-version.yml
48+
49+
#=============================================================================
50+
# Build the C++ Client
51+
#=============================================================================
52+
cpp_job:
53+
name: Build C++ client on Ubuntu or fetch from cache
54+
runs-on: ubuntu-24.04
55+
defaults:
56+
run:
57+
shell: bash
58+
59+
steps:
60+
- name: Check out this repo
61+
uses: actions/checkout@v6
62+
63+
# Try to fetch the whole installation directory (the final product of the
64+
# build) from the Github Cache. The key to this cache is derived
65+
# from the hash of the contents of the files in the cpp-client and proto
66+
# directories. If this fetch is successful, we skip over all the build
67+
# work and just copy this cached install directory to the output artifact
68+
# for this run.
69+
70+
- name: Restore the cached installation directory, or prepare a new one
71+
id: cache-cpp-install
72+
uses: actions/cache@v5
73+
with:
74+
path: ${{ env.DHINSTALL }}
75+
key: ${{ runner.os }}-cpp-client-${{ hashFiles('cpp-client/**', 'proto/**') }}
76+
77+
# [Note: skip this step if cache-cpp-install was successful]
78+
# Check out vcpkg
79+
80+
- name: Check out vcpkg
81+
id: checkout-vcpkg
82+
if: steps.cache-cpp-install.outputs.cache-hit != 'true'
83+
uses: actions/checkout@v6
84+
with:
85+
repository: microsoft/vcpkg
86+
path: vcpkg
87+
88+
# [Note: skip this step if cache-cpp-install was successful]
89+
# Bootstrap vcpkg
90+
91+
- name: Bootstrap vcpkg
92+
if: steps.cache-cpp-install.outputs.cache-hit != 'true'
93+
run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.sh
94+
95+
# [Note: skip this step if cache-cpp-install was successful]
96+
# Get mono so that we can run nuget. This is because the nuget.exe that
97+
# ships with vcpkg is an old .NET Framework binary (Windows format), and
98+
# only mono knows how to run such a thing on Linux. Some day the maintainers
99+
# will fix this so it runs with the cross-platform "dotnet".
100+
101+
- name: Get mono so that we can run nuget
102+
if: steps.cache-cpp-install.outputs.cache-hit != 'true'
103+
run: sudo apt install mono-complete
104+
105+
# [Note: skip this step if cache-cpp-install was successful]
106+
# Adapted from
107+
# https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages?pivots=linux-runner
108+
109+
# [Note: skip this step if cache-cpp-install was successful]
110+
- name: "Configure nuget to know how to download (or upload) each cached package that we need (or create)"
111+
if: steps.cache-cpp-install.outputs.cache-hit != 'true'
112+
run: |
113+
vcpkg_exe=${{ github.workspace }}/vcpkg/vcpkg
114+
nuget_dotnet=$($vcpkg_exe fetch nuget | tail -n 1)
115+
mono $nuget_dotnet \
116+
sources add \
117+
-Source "${{ env.GH_PACKAGES_FEED }}" \
118+
-StorePasswordInClearText \
119+
-Name GitHubPackages \
120+
-UserName "${{ env.GH_PACKAGES_USERNAME }}" \
121+
-Password "${{ secrets.GITHUB_TOKEN }}"
122+
mono $nuget_dotnet \
123+
setapikey "${{ secrets.GITHUB_TOKEN }}" \
124+
-Source "${{ env.GH_PACKAGES_FEED }}"
125+
126+
# [Note: skip this step if cache-cpp-install was successful]
127+
# 1. Invoke vcpkg to:
128+
# a) Try to fetch each of the dependent libraries from the nuget cache
129+
# b) If not found, build them and insert them in the nuget cache
130+
# 2. Configure cmake to prepare for the build/install step, which comes next
131+
132+
- name: build packages and configure cmake. Cached packages will be found, or stored in github's nuget repo
133+
if: steps.cache-cpp-install.outputs.cache-hit != 'true'
134+
env:
135+
VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite"
136+
run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DVCPKG_TARGET_TRIPLET=x64-linux-dynamic-release -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets
137+
138+
# [Note: skip this step if cache-cpp-install was successful]
139+
# Build our executables and install them in the configured install directory {{ env.DHINSTALL }}
140+
141+
- name: Run cmake build and install
142+
if: steps.cache-cpp-install.outputs.cache-hit != 'true'
143+
run: cmake --build cpp-client/deephaven/build --config RelWithDebInfo --target install --parallel
144+
145+
# Upload results as build artifact. Note: if cache-cpp-install was
146+
# then none of the intervening steps were executed. In that case,
147+
# this effectively just copies the cache to the build artifact.
148+
149+
- name: Upload install directory as build artifact
150+
id: upload-install-directory
151+
uses: actions/upload-artifact@v7
152+
with:
153+
name: cpp-client-install
154+
path: ${{ env.DHINSTALL }}
155+
156+
#=============================================================================
157+
# Build the Python Static Client
158+
#=============================================================================
159+
160+
python_static_job:
161+
name: Build Python static client on Ubuntu
162+
runs-on: ubuntu-24.04
163+
needs: [version_job]
164+
defaults:
165+
run:
166+
shell: bash
167+
168+
steps:
169+
- name: Check out this repo
170+
uses: actions/checkout@v6
171+
172+
- name: Install Python
173+
uses: actions/setup-python@v6
174+
with:
175+
python-version: '3.10'
176+
177+
- name: Install the 'wheel' package
178+
run: pip3 install wheel
179+
180+
- name: Install requirements-dev.txt
181+
run: |
182+
cd ./py/client
183+
pip3 install -r requirements-dev.txt
184+
185+
- name: Run setup.py
186+
env:
187+
DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }}
188+
run: |
189+
cd ./py/client
190+
python setup.py bdist_wheel
191+
192+
- name: Upload static .whl file as build artifact, to be used directly or by dependent clients like Python ticking
193+
uses: actions/upload-artifact@v7
194+
with:
195+
name: py-static-wheel
196+
path: ./py/client/dist/*.whl
197+
198+
#=============================================================================
199+
# Build the Python Ticking Client
200+
#=============================================================================
201+
202+
python_ticking_job:
203+
name: Build Python ticking client on Ubuntu
204+
runs-on: ubuntu-24.04
205+
needs: [cpp_job, python_static_job, version_job]
206+
defaults:
207+
run:
208+
shell: bash
209+
210+
steps:
211+
- name: Check out this repo
212+
uses: actions/checkout@v6
213+
214+
- name: Get the C++ installation as a build artifact
215+
uses: actions/download-artifact@v8
216+
with:
217+
name: cpp-client-install
218+
path: ${{ env.DHINSTALL }}
219+
220+
- name: Get the Python static installation as a build artifact
221+
uses: actions/download-artifact@v8
222+
with:
223+
name: py-static-wheel
224+
path: ./py/client/dist/
225+
226+
- name: Install Python
227+
uses: actions/setup-python@v6
228+
with:
229+
python-version: '3.10'
230+
231+
- name: Install the 'wheel' and 'cython' packages
232+
run: pip3 install wheel cython
233+
234+
- name: Install requirements-dev.txt
235+
run: |
236+
cd ./py/client
237+
pip3 install -r requirements-dev.txt
238+
239+
- name: Install the static client
240+
run: pip3 install --force --no-deps ./py/client/dist/*.whl
241+
242+
- name: Build cython code
243+
env:
244+
DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }}
245+
CPPFLAGS: -I${{ env.DHINSTALL }}/include
246+
LDFLAGS: -L${{ env.DHINSTALL }}/lib
247+
run: |
248+
cd ./py/client-ticking
249+
python setup.py build_ext -i
250+
251+
- name: Run setup.py
252+
env:
253+
DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }}
254+
run: |
255+
cd ./py/client-ticking
256+
python setup.py bdist_wheel
257+
258+
- name: Upload ticking .whl file as build artifact
259+
uses: actions/upload-artifact@v7
260+
with:
261+
name: py-ticking-wheel
262+
path: ./py/client-ticking/dist/*.whl

0 commit comments

Comments
 (0)