Skip to content

update caching and ci behaviour #636

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Shfdis merged 4 commits into from
Jul 6, 2026
Merged

update caching and ci behaviour #636

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3765f16
update caching and ci behaviour
Jul 2, 2026
1fd56c1
.deb package caching
Jul 2, 2026
458e71b
fix review issues
Jul 3, 2026
4c31153
fix test flap
Jul 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 33 additions & 1 deletion .github/workflows/coverage.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,14 +47,29 @@ jobs:
with:
submodules: true

- name: Prepare coverage cache key
id: coverage-cache-key
shell: bash
run: |
echo "prefix=ubuntu-22.04-coverage-gcc-${{ hashFiles('CMakePresets.json', '**/CMakeLists.txt', 'cmake/**') }}" >> "$GITHUB_OUTPUT"

- name: Restore coverage build cache
id: coverage-build-cache
uses: actions/cache/restore@v4
with:
path: build
key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: |
${{ steps.coverage-cache-key.outputs.prefix }}-
ubuntu-22.04-coverage-gcc-

- name: Install dependencies
uses: ./.github/actions/prepare_vm

- name: Configure and build with coverage
shell: bash
run: |
mkdir -p build
rm -rf build/*
cmake --preset coverage-test-gcc
cmake --build build -j"$(nproc)"

Expand All @@ -67,6 +82,9 @@ jobs:
FLAKY_REGEX='(ManyMessages|DiscoveryHang|DescribeHang)'
EXCLUDE_REGEX="${IAM_REGEX}|${FLAKY_REGEX}"

find build -type f \( -name '*.gcda' -o -name '*.gcov' \) -delete
rm -rf build/coverage

ctest -j1 --preset coverage-all -E "${EXCLUDE_REGEX}" --output-on-failure

IAM_CTEST_JOBS=1 ./.github/scripts/run_iam_integration_tests.sh
Expand Down Expand Up @@ -96,3 +114,17 @@ jobs:
name: coverage-report
path: build/coverage/
retention-days: 14

- name: Clean coverage data before cache save
if: github.event_name != 'pull_request' && success()
shell: bash
run: |
find build -type f \( -name '*.gcda' -o -name '*.gcov' \) -delete
rm -rf build/coverage

- name: Save coverage build cache
if: github.event_name != 'pull_request' && success() && steps.coverage-build-cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: build
key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
77 changes: 21 additions & 56 deletions .github/workflows/release_publish.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,68 +39,33 @@ jobs:
submodules: true
ref: ${{ steps.tag.outputs.ref }}

- name: Prepare Debian package cache key
id: deb-package-cache-key
shell: bash
run: |
echo "prefix=ubuntu-24.04-deb-packages-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'contrib/**', 'include/**', 'library/**', 'plugins/**', 'scripts/build_cpack_deb_packages.sh', 'scripts/generate-debian-directory.sh', 'scripts/googleapis_deb/**', 'src/**', 'third_party/api-common-protos/**', 'tools/**', 'util/**') }}" >> "$GITHUB_OUTPUT"

- name: Restore Debian package build cache
uses: actions/cache/restore@v4
with:
path: |
.deb-ccache
build_googleapis_deb
build-deb
key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: |
${{ steps.deb-package-cache-key.outputs.prefix }}-
ubuntu-24.04-deb-packages-

- name: Build .deb packages in Ubuntu 24.04 container
shell: bash
run: |
mkdir -p artifacts
mkdir -p artifacts .deb-ccache build_googleapis_deb build-deb
docker run --rm --network host \
-e CCACHE_DIR=/source/.deb-ccache \
-v "$PWD:/source" \
ubuntu:24.04 \
bash -c '
set -e
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get install -y \
build-essential \
cmake \
pkg-config \
git \
libidn11-dev \
libssl-dev \
zlib1g-dev \
libprotobuf-dev \
protobuf-compiler \
libgrpc++-dev \
protobuf-compiler-grpc \
libbrotli-dev \
liblz4-dev \
libzstd-dev \
libbz2-dev \
libxxhash-dev \
libsnappy-dev \
libdouble-conversion-dev \
libgtest-dev \
libre2-dev \
libc-ares-dev \
rapidjson-dev \
python3 \
python3-six \
ragel \
yasm

cd /source
cmake -S scripts/googleapis_deb -B build_googleapis_deb -DCMAKE_INSTALL_PREFIX=/usr/share/yandex
cmake --build build_googleapis_deb -j$(nproc)
cmake --build build_googleapis_deb --target package
dpkg -i build_googleapis_deb/*.deb

./scripts/generate-debian-directory.sh
cmake -S . -B build-deb \
-DCMAKE_BUILD_TYPE=Release \
-DYDB_SDK_INSTALL=ON \
-DYDB_SDK_EXAMPLES=OFF \
-DYDB_SDK_TESTS=OFF \
-DYDB_SDK_ENABLE_OTEL_METRICS=ON \
-DYDB_SDK_ENABLE_OTEL_TRACE=ON \
-DBUILD_SHARED_LIBS=OFF \
-DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \
-DCMAKE_INSTALL_PREFIX=/usr/share/yandex \
-DCMAKE_PREFIX_PATH="/usr/share/yandex"
cmake --build build-deb --target package -j$(nproc)

cp build_googleapis_deb/*.deb /source/artifacts/
cp build-deb/*.deb /source/artifacts/
'
bash /source/scripts/build_cpack_deb_packages.sh /source/artifacts

- name: Smoke-test generated .deb packages
shell: bash
Expand Down
187 changes: 110 additions & 77 deletions .github/workflows/tests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ concurrency:
jobs:
unit:
concurrency:
group: unit-${{ github.ref }}-${{ matrix.os }}-${{ matrix.compiler }}
group: unit-${{ github.ref }}-ubuntu-22.04-${{ matrix.compiler }}
cancel-in-progress: true
strategy:
fail-fast: false
Expand All @@ -37,10 +37,9 @@ jobs:
submodules: true
- name: Prepare ccache timestamp
id: ccache_cache_timestamp
shell: cmake -P {0}
shell: bash
run: |
string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
message("::set-output name=timestamp::${current_date}")
echo "timestamp=$(date -u +%Y-%m-%d-%H-%M-%S)" >> "$GITHUB_OUTPUT"
- name: Restore cache files
uses: actions/cache/restore@v4
with:
Expand All @@ -59,7 +58,61 @@ jobs:
run: |
ctest -j$(nproc) --preset unit

integration-build:
concurrency:
group: integration-build-${{ github.ref }}-${{ matrix.compiler }}
cancel-in-progress: true
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
compiler: [clang, gcc]
steps:
- name: Checkout PR
uses: actions/checkout@v4
if: github.event.pull_request.head.sha != ''
with:
submodules: true
ref: ${{ github.event.pull_request.head.sha }}
- name: Checkout
uses: actions/checkout@v4
if: github.event.pull_request.head.sha == ''
with:
submodules: true
Comment thread
Shfdis marked this conversation as resolved.
Outdated
- name: Prepare ccache timestamp
id: ccache_cache_timestamp
shell: bash
run: |
echo "timestamp=$(date -u +%Y-%m-%d-%H-%M-%S)" >> "$GITHUB_OUTPUT"
- name: Restore cache files
uses: actions/cache/restore@v4
with:
path: ~/.ccache
Comment thread
Shfdis marked this conversation as resolved.
Outdated
key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }}
restore-keys: |
ubuntu-22.04-ccache-${{ matrix.compiler }}-
- name: Install dependencies
uses: ./.github/actions/prepare_vm
- name: Build
uses: ./.github/actions/build
with:
compiler: ${{ matrix.compiler }}
- name: Package integration build
shell: bash
run: |
tar -C build -czf "integration-build-${{ matrix.compiler }}.tar.gz" .
tar -C "$HOME" -czf "integration-deps-${{ matrix.compiler }}.tar.gz" ydb_deps
- name: Upload integration build
uses: actions/upload-artifact@v4
with:
name: integration-${{ matrix.compiler }}
path: |
integration-build-${{ matrix.compiler }}.tar.gz
integration-deps-${{ matrix.compiler }}.tar.gz
retention-days: 3

integration:
needs: integration-build
concurrency:
group: integration-${{ github.ref }}-${{ matrix.ydb-version }}-${{ matrix.compiler }}
cancel-in-progress: true
Expand Down Expand Up @@ -95,25 +148,27 @@ jobs:
if: github.event.pull_request.head.sha == ''
with:
submodules: true
- name: Prepare ccache timestamp
id: ccache_cache_timestamp
shell: cmake -P {0}
- name: Install CMake
uses: lukka/get-cmake@v3.27.7
- name: Install runtime dependencies
shell: bash
run: |
string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
message("::set-output name=timestamp::${current_date}")
- name: Restore cache files
uses: actions/cache/restore@v4
sudo add-apt-repository ppa:ubuntu-toolchain-r/test
Comment thread
Shfdis marked this conversation as resolved.
Outdated
sudo apt-get -y update
sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \
rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \
libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13
- name: Download integration build
uses: actions/download-artifact@v4
with:
path: ~/.ccache
key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }}
restore-keys: |
ubuntu-22.04-ccache-${{ matrix.compiler }}-
- name: Install dependencies
uses: ./.github/actions/prepare_vm
- name: Build
uses: ./.github/actions/build
with:
compiler: ${{ matrix.compiler }}
name: integration-${{ matrix.compiler }}
path: .
- name: Unpack integration build
shell: bash
run: |
mkdir -p build
tar -C build -xzf "integration-build-${{ matrix.compiler }}.tar.gz"
tar -C "$HOME" -xzf "integration-deps-${{ matrix.compiler }}.tar.gz"
- name: Test
shell: bash
run: |
Expand Down Expand Up @@ -157,10 +212,9 @@ jobs:

- name: Prepare ccache timestamp
id: ccache_cache_timestamp
shell: cmake -P {0}
shell: bash
run: |
string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
message("::set-output name=timestamp::${current_date}")
echo "timestamp=$(date -u +%Y-%m-%d-%H-%M-%S)" >> "$GITHUB_OUTPUT"

- name: Restore cache files
uses: actions/cache/restore@v4
Expand Down Expand Up @@ -208,72 +262,51 @@ jobs:
with:
submodules: true

- name: Prepare Debian package cache key
id: deb-package-cache-key
shell: bash
run: |
echo "prefix=ubuntu-24.04-deb-packages-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'contrib/**', 'include/**', 'library/**', 'plugins/**', 'scripts/build_cpack_deb_packages.sh', 'scripts/generate-debian-directory.sh', 'scripts/googleapis_deb/**', 'src/**', 'third_party/api-common-protos/**', 'tools/**', 'util/**') }}" >> "$GITHUB_OUTPUT"

- name: Validate dpkg-buildpackage
shell: bash
run: |
./scripts/test_dpkg_buildpackage.sh

- name: Restore Debian package build cache
id: deb-package-cache
uses: actions/cache/restore@v4
with:
path: |
.deb-ccache
build_googleapis_deb
build-deb
key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: |
${{ steps.deb-package-cache-key.outputs.prefix }}-
ubuntu-24.04-deb-packages-

- name: Build CPack .deb packages
shell: bash
run: |
mkdir -p .deb-ccache build_googleapis_deb build-deb
docker run --rm --network host \
-e CCACHE_DIR=/source/.deb-ccache \
-v "$PWD:/source" \
ubuntu:24.04 \
bash -c '
set -e
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get install -y \
build-essential \
cmake \
pkg-config \
git \
libidn11-dev \
libssl-dev \
zlib1g-dev \
libprotobuf-dev \
protobuf-compiler \
libgrpc++-dev \
protobuf-compiler-grpc \
libbrotli-dev \
liblz4-dev \
libzstd-dev \
libbz2-dev \
libxxhash-dev \
libsnappy-dev \
libdouble-conversion-dev \
libgtest-dev \
libre2-dev \
libc-ares-dev \
rapidjson-dev \
python3 \
python3-six \
ragel \
yasm

cd /source
cmake -S scripts/googleapis_deb -B build_googleapis_deb -DCMAKE_INSTALL_PREFIX=/usr/share/yandex
cmake --build build_googleapis_deb -j$(nproc)
cmake --build build_googleapis_deb --target package
dpkg -i build_googleapis_deb/*.deb

./scripts/generate-debian-directory.sh
cmake -S . -B build-deb \
-DCMAKE_BUILD_TYPE=Release \
-DYDB_SDK_INSTALL=ON \
-DYDB_SDK_EXAMPLES=OFF \
-DYDB_SDK_TESTS=OFF \
-DYDB_SDK_ENABLE_OTEL_METRICS=ON \
-DYDB_SDK_ENABLE_OTEL_TRACE=ON \
-DBUILD_SHARED_LIBS=OFF \
-DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \
-DCMAKE_INSTALL_PREFIX=/usr/share/yandex \
-DCMAKE_PREFIX_PATH="/usr/share/yandex"
cmake --build build-deb --target package -j$(nproc)
cp build_googleapis_deb/*.deb build-deb/
'
bash /source/scripts/build_cpack_deb_packages.sh /source/build-deb

- name: Smoke-test generated .deb packages
shell: bash
run: |
./scripts/test_deb_packages.sh build-deb

- name: Save Debian package build cache
if: github.event_name != 'pull_request' && success() && steps.deb-package-cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: |
.deb-ccache
build_googleapis_deb
build-deb
key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
Loading
Loading