diff --git a/.github/actions/prepare_vm/action.yaml b/.github/actions/prepare_vm/action.yaml index a953ed8438..35819ab630 100644 --- a/.github/actions/prepare_vm/action.yaml +++ b/.github/actions/prepare_vm/action.yaml @@ -1,12 +1,18 @@ name: Prepare VM for YDB SDK build description: Install required packages +inputs: + mode: + description: "Setup scope: build installs full build dependencies; runtime installs only test runtime dependencies." + required: false + default: build + runs: using: "composite" steps: - name: Install CMake and Ninja uses: lukka/get-cmake@v3.27.7 - - name: Install dependencies + - name: Install runtime dependencies shell: bash run: | sudo add-apt-repository ppa:ubuntu-toolchain-r/test @@ -15,6 +21,13 @@ runs: rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \ libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13 + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 10000 + + - name: Install build dependencies + if: ${{ inputs.mode == 'build' }} + shell: bash + run: | # Install ccache (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \ sudo tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache) @@ -30,9 +43,6 @@ runs: sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-16 10000 sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-16 10000 - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000 - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 10000 - # Install abseil-cpp wget -O abseil-cpp-20230802.0.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz tar -xvzf abseil-cpp-20230802.0.tar.gz diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index caeda7f23f..6fb34b974e 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -47,6 +47,22 @@ jobs: with: submodules: true + - name: Prepare coverage cache key + id: coverage-cache-key + shell: bash + run: | + echo "prefix=ubuntu-22.04-coverage-gcc-${{ hashFiles('CMakePresets.json', '**/CMakeLists.txt', 'cmake/**') }}" >> "$GITHUB_OUTPUT" + + - name: Restore coverage build cache + id: coverage-build-cache + uses: actions/cache/restore@v4 + with: + path: build + key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + ${{ steps.coverage-cache-key.outputs.prefix }}- + ubuntu-22.04-coverage-gcc- + - name: Install dependencies uses: ./.github/actions/prepare_vm @@ -54,7 +70,6 @@ jobs: shell: bash run: | mkdir -p build - rm -rf build/* cmake --preset coverage-test-gcc cmake --build build -j"$(nproc)" @@ -67,6 +82,9 @@ jobs: FLAKY_REGEX='(ManyMessages|DiscoveryHang|DescribeHang)' EXCLUDE_REGEX="${IAM_REGEX}|${FLAKY_REGEX}" + find build -type f \( -name '*.gcda' -o -name '*.gcov' \) -delete + rm -rf build/coverage + ctest -j1 --preset coverage-all -E "${EXCLUDE_REGEX}" --output-on-failure IAM_CTEST_JOBS=1 ./.github/scripts/run_iam_integration_tests.sh @@ -96,3 +114,17 @@ jobs: name: coverage-report path: build/coverage/ retention-days: 14 + + - name: Clean coverage data before cache save + if: github.event_name != 'pull_request' && success() + shell: bash + run: | + find build -type f \( -name '*.gcda' -o -name '*.gcov' \) -delete + rm -rf build/coverage + + - name: Save coverage build cache + if: github.event_name != 'pull_request' && success() && steps.coverage-build-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: build + key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml index 6ab6dd96c1..d471725043 100644 --- a/.github/workflows/release_publish.yaml +++ b/.github/workflows/release_publish.yaml @@ -39,68 +39,33 @@ jobs: submodules: true ref: ${{ steps.tag.outputs.ref }} + - name: Prepare Debian package cache key + id: deb-package-cache-key + shell: bash + run: | + echo "prefix=ubuntu-24.04-deb-packages-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'contrib/**', 'include/**', 'library/**', 'plugins/**', 'scripts/build_cpack_deb_packages.sh', 'scripts/generate-debian-directory.sh', 'scripts/googleapis_deb/**', 'src/**', 'third_party/api-common-protos/**', 'tools/**', 'util/**') }}" >> "$GITHUB_OUTPUT" + + - name: Restore Debian package build cache + uses: actions/cache/restore@v4 + with: + path: | + .deb-ccache + build_googleapis_deb + build-deb + key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + ${{ steps.deb-package-cache-key.outputs.prefix }}- + ubuntu-24.04-deb-packages- + - name: Build .deb packages in Ubuntu 24.04 container shell: bash run: | - mkdir -p artifacts + mkdir -p artifacts .deb-ccache build_googleapis_deb build-deb docker run --rm --network host \ + -e CCACHE_DIR=/source/.deb-ccache \ -v "$PWD:/source" \ ubuntu:24.04 \ - bash -c ' - set -e - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y \ - build-essential \ - cmake \ - pkg-config \ - git \ - libidn11-dev \ - libssl-dev \ - zlib1g-dev \ - libprotobuf-dev \ - protobuf-compiler \ - libgrpc++-dev \ - protobuf-compiler-grpc \ - libbrotli-dev \ - liblz4-dev \ - libzstd-dev \ - libbz2-dev \ - libxxhash-dev \ - libsnappy-dev \ - libdouble-conversion-dev \ - libgtest-dev \ - libre2-dev \ - libc-ares-dev \ - rapidjson-dev \ - python3 \ - python3-six \ - ragel \ - yasm - - cd /source - cmake -S scripts/googleapis_deb -B build_googleapis_deb -DCMAKE_INSTALL_PREFIX=/usr/share/yandex - cmake --build build_googleapis_deb -j$(nproc) - cmake --build build_googleapis_deb --target package - dpkg -i build_googleapis_deb/*.deb - - ./scripts/generate-debian-directory.sh - cmake -S . -B build-deb \ - -DCMAKE_BUILD_TYPE=Release \ - -DYDB_SDK_INSTALL=ON \ - -DYDB_SDK_EXAMPLES=OFF \ - -DYDB_SDK_TESTS=OFF \ - -DYDB_SDK_ENABLE_OTEL_METRICS=ON \ - -DYDB_SDK_ENABLE_OTEL_TRACE=ON \ - -DBUILD_SHARED_LIBS=OFF \ - -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \ - -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ - -DCMAKE_PREFIX_PATH="/usr/share/yandex" - cmake --build build-deb --target package -j$(nproc) - - cp build_googleapis_deb/*.deb /source/artifacts/ - cp build-deb/*.deb /source/artifacts/ - ' + bash /source/scripts/build_cpack_deb_packages.sh /source/artifacts - name: Smoke-test generated .deb packages shell: bash diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index bf88544d10..54e79c108d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -12,9 +12,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true jobs: - unit: + build-and-unit: concurrency: - group: unit-${{ github.ref }}-${{ matrix.os }}-${{ matrix.compiler }} + group: build-and-unit-${{ github.ref }}-ubuntu-22.04-${{ matrix.compiler }} cancel-in-progress: true strategy: fail-fast: false @@ -37,10 +37,9 @@ jobs: submodules: true - name: Prepare ccache timestamp id: ccache_cache_timestamp - shell: cmake -P {0} + shell: bash run: | - string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) - message("::set-output name=timestamp::${current_date}") + echo "timestamp=$(date -u +%Y-%m-%d-%H-%M-%S)" >> "$GITHUB_OUTPUT" - name: Restore cache files uses: actions/cache/restore@v4 with: @@ -58,8 +57,30 @@ jobs: shell: bash run: | ctest -j$(nproc) --preset unit + - name: Package integration build + shell: bash + run: | + tar -C build \ + --exclude='*.o' \ + --exclude='*.obj' \ + --exclude='*.a' \ + --exclude='*.d' \ + --exclude='*.rsp' \ + --exclude='.ninja_deps' \ + --exclude='.ninja_log' \ + -czf "integration-build-${{ matrix.compiler }}.tar.gz" . + tar -C "$HOME" -czf "integration-deps-${{ matrix.compiler }}.tar.gz" ydb_deps + - name: Upload integration build + uses: actions/upload-artifact@v4 + with: + name: integration-${{ matrix.compiler }} + path: | + integration-build-${{ matrix.compiler }}.tar.gz + integration-deps-${{ matrix.compiler }}.tar.gz + retention-days: 3 integration: + needs: build-and-unit concurrency: group: integration-${{ github.ref }}-${{ matrix.ydb-version }}-${{ matrix.compiler }} cancel-in-progress: true @@ -95,25 +116,21 @@ jobs: if: github.event.pull_request.head.sha == '' with: submodules: true - - name: Prepare ccache timestamp - id: ccache_cache_timestamp - shell: cmake -P {0} - run: | - string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) - message("::set-output name=timestamp::${current_date}") - - name: Restore cache files - uses: actions/cache/restore@v4 - with: - path: ~/.ccache - key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} - restore-keys: | - ubuntu-22.04-ccache-${{ matrix.compiler }}- - - name: Install dependencies + - name: Install runtime dependencies uses: ./.github/actions/prepare_vm - - name: Build - uses: ./.github/actions/build with: - compiler: ${{ matrix.compiler }} + mode: runtime + - name: Download integration build + uses: actions/download-artifact@v4 + with: + name: integration-${{ matrix.compiler }} + path: . + - name: Unpack integration build + shell: bash + run: | + mkdir -p build + tar -C build -xzf "integration-build-${{ matrix.compiler }}.tar.gz" + tar -C "$HOME" -xzf "integration-deps-${{ matrix.compiler }}.tar.gz" - name: Test shell: bash run: | @@ -157,10 +174,9 @@ jobs: - name: Prepare ccache timestamp id: ccache_cache_timestamp - shell: cmake -P {0} + shell: bash run: | - string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) - message("::set-output name=timestamp::${current_date}") + echo "timestamp=$(date -u +%Y-%m-%d-%H-%M-%S)" >> "$GITHUB_OUTPUT" - name: Restore cache files uses: actions/cache/restore@v4 @@ -208,72 +224,51 @@ jobs: with: submodules: true + - name: Prepare Debian package cache key + id: deb-package-cache-key + shell: bash + run: | + echo "prefix=ubuntu-24.04-deb-packages-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'contrib/**', 'include/**', 'library/**', 'plugins/**', 'scripts/build_cpack_deb_packages.sh', 'scripts/generate-debian-directory.sh', 'scripts/googleapis_deb/**', 'src/**', 'third_party/api-common-protos/**', 'tools/**', 'util/**') }}" >> "$GITHUB_OUTPUT" + - name: Validate dpkg-buildpackage shell: bash run: | ./scripts/test_dpkg_buildpackage.sh + - name: Restore Debian package build cache + id: deb-package-cache + uses: actions/cache/restore@v4 + with: + path: | + .deb-ccache + build_googleapis_deb + build-deb + key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + ${{ steps.deb-package-cache-key.outputs.prefix }}- + ubuntu-24.04-deb-packages- + - name: Build CPack .deb packages shell: bash run: | + mkdir -p .deb-ccache build_googleapis_deb build-deb docker run --rm --network host \ + -e CCACHE_DIR=/source/.deb-ccache \ -v "$PWD:/source" \ ubuntu:24.04 \ - bash -c ' - set -e - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y \ - build-essential \ - cmake \ - pkg-config \ - git \ - libidn11-dev \ - libssl-dev \ - zlib1g-dev \ - libprotobuf-dev \ - protobuf-compiler \ - libgrpc++-dev \ - protobuf-compiler-grpc \ - libbrotli-dev \ - liblz4-dev \ - libzstd-dev \ - libbz2-dev \ - libxxhash-dev \ - libsnappy-dev \ - libdouble-conversion-dev \ - libgtest-dev \ - libre2-dev \ - libc-ares-dev \ - rapidjson-dev \ - python3 \ - python3-six \ - ragel \ - yasm - - cd /source - cmake -S scripts/googleapis_deb -B build_googleapis_deb -DCMAKE_INSTALL_PREFIX=/usr/share/yandex - cmake --build build_googleapis_deb -j$(nproc) - cmake --build build_googleapis_deb --target package - dpkg -i build_googleapis_deb/*.deb - - ./scripts/generate-debian-directory.sh - cmake -S . -B build-deb \ - -DCMAKE_BUILD_TYPE=Release \ - -DYDB_SDK_INSTALL=ON \ - -DYDB_SDK_EXAMPLES=OFF \ - -DYDB_SDK_TESTS=OFF \ - -DYDB_SDK_ENABLE_OTEL_METRICS=ON \ - -DYDB_SDK_ENABLE_OTEL_TRACE=ON \ - -DBUILD_SHARED_LIBS=OFF \ - -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \ - -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ - -DCMAKE_PREFIX_PATH="/usr/share/yandex" - cmake --build build-deb --target package -j$(nproc) - cp build_googleapis_deb/*.deb build-deb/ - ' + bash /source/scripts/build_cpack_deb_packages.sh /source/build-deb - name: Smoke-test generated .deb packages shell: bash run: | ./scripts/test_deb_packages.sh build-deb + + - name: Save Debian package build cache + if: github.event_name != 'pull_request' && success() && steps.deb-package-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: | + .deb-ccache + build_googleapis_deb + build-deb + key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/.github/workflows/warmup_cache.yaml b/.github/workflows/warmup_cache.yaml index 5c2673c514..be353eeea6 100644 --- a/.github/workflows/warmup_cache.yaml +++ b/.github/workflows/warmup_cache.yaml @@ -17,7 +17,13 @@ jobs: uses: actions/checkout@v4 with: submodules: true + - name: Prepare ccache timestamp + id: ccache_cache_timestamp + shell: bash + run: | + echo "timestamp=$(date -u +%Y-%m-%d-%H-%M-%S)" >> "$GITHUB_OUTPUT" - name: Restore cache files + id: ccache-restore uses: actions/cache/restore@v4 with: path: ~/.ccache @@ -26,20 +32,96 @@ jobs: ubuntu-22.04-ccache-${{ matrix.compiler }}- - name: Install dependencies uses: ./.github/actions/prepare_vm - - name: Prepare ccache timestamp - id: ccache_cache_timestamp - shell: cmake -P {0} - run: | - string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) - message("::set-output name=timestamp::${current_date}") - - name: Cache files - uses: actions/cache@v4 + - name: Build + uses: ./.github/actions/build + with: + compiler: ${{ matrix.compiler }} + - name: Save cache files + if: steps.ccache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 with: path: ~/.ccache key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} + + coverage: + name: Build coverage binaries and cache artifacts + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + - name: Prepare coverage cache key + id: coverage-cache-key + shell: bash + run: | + echo "prefix=ubuntu-22.04-coverage-gcc-${{ hashFiles('CMakePresets.json', '**/CMakeLists.txt', 'cmake/**') }}" >> "$GITHUB_OUTPUT" + - name: Restore coverage build cache + id: coverage-build-cache + uses: actions/cache/restore@v4 + with: + path: build + key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | - ubuntu-22.04-ccache-${{ matrix.compiler }}- - - name: Build - uses: ./.github/actions/build + ${{ steps.coverage-cache-key.outputs.prefix }}- + ubuntu-22.04-coverage-gcc- + - name: Install dependencies + uses: ./.github/actions/prepare_vm + - name: Configure and build with coverage + shell: bash + run: | + mkdir -p build + cmake --preset coverage-test-gcc + cmake --build build -j"$(nproc)" + find build -type f \( -name '*.gcda' -o -name '*.gcov' \) -delete + rm -rf build/coverage + - name: Save coverage build cache + if: steps.coverage-build-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 with: - compiler: ${{ matrix.compiler }} + path: build + key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + + deb-packages: + name: Build Debian packages and cache artifacts + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + - name: Prepare Debian package cache key + id: deb-package-cache-key + shell: bash + run: | + echo "prefix=ubuntu-24.04-deb-packages-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'contrib/**', 'include/**', 'library/**', 'plugins/**', 'scripts/build_cpack_deb_packages.sh', 'scripts/generate-debian-directory.sh', 'scripts/googleapis_deb/**', 'src/**', 'third_party/api-common-protos/**', 'tools/**', 'util/**') }}" >> "$GITHUB_OUTPUT" + - name: Restore Debian package build cache + id: deb-package-cache + uses: actions/cache/restore@v4 + with: + path: | + .deb-ccache + build_googleapis_deb + build-deb + key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + ${{ steps.deb-package-cache-key.outputs.prefix }}- + ubuntu-24.04-deb-packages- + - name: Build CPack .deb packages + shell: bash + run: | + mkdir -p .deb-ccache build_googleapis_deb build-deb + docker run --rm --network host \ + -e CCACHE_DIR=/source/.deb-ccache \ + -v "$PWD:/source" \ + ubuntu:24.04 \ + bash /source/scripts/build_cpack_deb_packages.sh /source/build-deb + - name: Save Debian package build cache + if: steps.deb-package-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: | + .deb-ccache + build_googleapis_deb + build-deb + key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/.gitignore b/.gitignore index cf184074ff..297b3bc31e 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,8 @@ __pycache__/ /build /build_* /build-* +/.deb-ccache/ +/artifacts/ .idea/ .vscode/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 2cf1bc9e66..8db73349aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,7 @@ option(YDB_SDK_TESTS "Build YDB C++ SDK tests" Off) option(YDB_SDK_EXAMPLES "Build YDB C++ SDK examples" On) option(YDB_SDK_ENABLE_OTEL_METRICS "Build OpenTelemetry metrics plugin" Off) option(YDB_SDK_ENABLE_OTEL_TRACE "Build OpenTelemetry trace plugin" Off) +option(YDB_CPP_SDK_SLO_USE_INSTALLED_SDK "Build only SLO workloads against an installed ydb-cpp-sdk package" Off) set(YDB_SDK_GOOGLE_COMMON_PROTOS_TARGET "" CACHE STRING "Name of cmake target preparing google common proto library") option(YDB_SDK_USE_RAPID_JSON "Search for rapid json library in system" ON) @@ -51,8 +52,26 @@ include(cmake/ccache.cmake) include(cmake/protobuf.cmake) include(cmake/external_libs.cmake) +set(YDB_SDK_TESTING_INCLUDED OFF) if (YDB_SDK_TESTS) include(cmake/testing.cmake) + set(YDB_SDK_TESTING_INCLUDED ON) +endif() + +if (YDB_CPP_SDK_SLO_USE_INSTALLED_SDK) + if (NOT YDB_SDK_TESTING_INCLUDED) + include(cmake/testing.cmake) + endif() + + find_package(ydb-cpp-sdk REQUIRED COMPONENTS Table Iam) + + add_subdirectory(tools) + add_subdirectory(contrib/libs) + add_subdirectory(library/cpp) + add_subdirectory(util) + add_subdirectory(tests/slo_workloads) + + return() endif() add_subdirectory(tools) diff --git a/Testing/Temporary/LastTest.log b/Testing/Temporary/LastTest.log deleted file mode 100644 index 7d298711f7..0000000000 --- a/Testing/Temporary/LastTest.log +++ /dev/null @@ -1,3 +0,0 @@ -Start testing: Jul 01 00:28 MSK ----------------------------------------------------------- -End testing: Jul 01 00:28 MSK diff --git a/scripts/build_cpack_deb_packages.sh b/scripts/build_cpack_deb_packages.sh new file mode 100755 index 0000000000..91f0b8941e --- /dev/null +++ b/scripts/build_cpack_deb_packages.sh @@ -0,0 +1,141 @@ +#!/bin/bash +set -euo pipefail + +SOURCE_DIR="${SOURCE_DIR:-/source}" +if [ ! -d "$SOURCE_DIR" ]; then + SOURCE_DIR=$(dirname "$(realpath "$0")")/.. + SOURCE_DIR=$(realpath "$SOURCE_DIR") +fi + +OUTPUT_DIR="${1:-build-deb}" + +cd "$SOURCE_DIR" +mkdir -p "$OUTPUT_DIR" +OUTPUT_DIR=$(realpath "$OUTPUT_DIR") + +export DEBIAN_FRONTEND=noninteractive +export CCACHE_DIR="${CCACHE_DIR:-/root/.ccache}" +mkdir -p "$CCACHE_DIR" + +if [ "${YDB_DEB_INSTALL_DEPS:-1}" = "1" ]; then + apt-get update + apt-get install -y --no-install-recommends \ + build-essential \ + ccache \ + cmake \ + pkg-config \ + git \ + libidn11-dev \ + libssl-dev \ + zlib1g-dev \ + libprotobuf-dev \ + protobuf-compiler \ + libgrpc++-dev \ + protobuf-compiler-grpc \ + libbrotli-dev \ + liblz4-dev \ + libzstd-dev \ + libbz2-dev \ + libxxhash-dev \ + libsnappy-dev \ + libdouble-conversion-dev \ + libgtest-dev \ + libre2-dev \ + libc-ares-dev \ + rapidjson-dev \ + python3 \ + python3-six \ + ragel \ + yasm +fi + +touch_existing_sources() { + for path in "$@"; do + if [ -e "$path" ]; then + find "$path" -type f -exec touch {} + + fi + done +} + +fix_output_permissions() { + local uid="${HOST_UID:-$(stat -c '%u' "$SOURCE_DIR" 2>/dev/null || true)}" + local gid="${HOST_GID:-$(stat -c '%g' "$SOURCE_DIR" 2>/dev/null || true)}" + local paths=() + + for path in "$@"; do + [ ! -e "$path" ] || paths+=("$path") + done + if [ "${#paths[@]}" -eq 0 ]; then + return + fi + + if [[ "$uid" =~ ^[0-9]+$ && "$gid" =~ ^[0-9]+$ ]] && chown -R "$uid:$gid" "${paths[@]}" 2>/dev/null; then + return + fi + + chmod -R a+rX "${paths[@]}" 2>/dev/null || true +} + +rm -f build_googleapis_deb/*.deb build-deb/*.deb "$OUTPUT_DIR"/*.deb 2>/dev/null || true + +if command -v ccache >/dev/null 2>&1; then + ccache --zero-stats >/dev/null || true + CMAKE_COMPILER_LAUNCHER_ARGS=( + -DCMAKE_C_COMPILER_LAUNCHER=ccache + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + ) +else + CMAKE_COMPILER_LAUNCHER_ARGS=() +fi + +# Restored build-cache archives can have outputs newer than the checkout. +# Touch package inputs so the incremental build cannot hide source changes. +touch_existing_sources scripts/googleapis_deb third_party/api-common-protos +cmake -S scripts/googleapis_deb -B build_googleapis_deb \ + -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ + "${CMAKE_COMPILER_LAUNCHER_ARGS[@]}" +cmake --build build_googleapis_deb -j"$(nproc)" +cmake --build build_googleapis_deb --target package +dpkg -i build_googleapis_deb/*.deb + +./scripts/generate-debian-directory.sh + +touch_existing_sources \ + CMakeLists.txt \ + cmake \ + contrib \ + include \ + library \ + plugins \ + scripts/build_cpack_deb_packages.sh \ + scripts/generate-debian-directory.sh \ + src \ + third_party \ + tools \ + util + +cmake -S . -B build-deb \ + -DCMAKE_BUILD_TYPE=Release \ + -DYDB_SDK_INSTALL=ON \ + -DYDB_SDK_EXAMPLES=OFF \ + -DYDB_SDK_TESTS=OFF \ + -DYDB_SDK_ENABLE_OTEL_METRICS=ON \ + -DYDB_SDK_ENABLE_OTEL_TRACE=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ + -DCMAKE_PREFIX_PATH="/usr/share/yandex" \ + "${CMAKE_COMPILER_LAUNCHER_ARGS[@]}" +cmake --build build-deb --target package -j"$(nproc)" + +cp -f build_googleapis_deb/*.deb "$OUTPUT_DIR"/ +if [ "$(realpath build-deb)" != "$OUTPUT_DIR" ]; then + cp -f build-deb/*.deb "$OUTPUT_DIR"/ +fi + +if command -v ccache >/dev/null 2>&1; then + ccache --show-stats || true +fi + +fix_output_permissions "$CCACHE_DIR" build_googleapis_deb build-deb "$OUTPUT_DIR" debian +ls -la "$OUTPUT_DIR" diff --git a/src/version.h b/src/version.h index be835d99f5..2c9931bca6 100644 --- a/src/version.h +++ b/src/version.h @@ -2,7 +2,7 @@ namespace NYdb { -inline const char* YDB_SDK_VERSION = "3.19.0"; +inline const char* YDB_SDK_VERSION = "3.20.0"; inline const char* YDB_CERTIFICATE_FILE_KEY = "ydb_root_ca_v3.pem"; } // namespace NYdb diff --git a/tests/slo_workloads/Dockerfile b/tests/slo_workloads/Dockerfile index f87be8f8df..5a5c7f2821 100644 --- a/tests/slo_workloads/Dockerfile +++ b/tests/slo_workloads/Dockerfile @@ -2,6 +2,7 @@ FROM ubuntu:22.04 ARG PRESET=release-test-clang +ARG SDK_PRESET=release-clang # ccache settings consumed by the configure/build steps below. The cache dir # is materialised by the BuildKit cache mount on those RUN steps; values @@ -19,6 +20,8 @@ RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \ echo 'Acquire::https::Timeout "60";' >> /etc/apt/apt.conf.d/80-retries ENV WGET_OPTS="--tries=5 --waitretry=15 --timeout=60 --retry-connrefused --retry-on-http-error=500,502,503,504" +ENV YDB_CPP_SDK_INSTALL_PREFIX=/opt/ydb-cpp-sdk +ENV YDB_DEPS_PREFIX_PATH="${YDB_CPP_SDK_INSTALL_PREFIX};/root/ydb_deps/absl;/root/ydb_deps/protobuf;/root/ydb_deps/grpc;/root/ydb_deps/brotli" # Install software-properties-common and add the gcc-13 PPA. # Acquire::Retries only retries HTTP errors; TCP connect timeouts to @@ -156,8 +159,27 @@ COPY . /ydb-cpp-sdk WORKDIR /ydb-cpp-sdk RUN rm -rf build +RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ + cmake --preset ${SDK_PRESET} \ + -DYDB_SDK_INSTALL=ON \ + -DYDB_SDK_EXAMPLES=OFF \ + -DYDB_SDK_TESTS=OFF \ + -DCMAKE_INSTALL_PREFIX=${YDB_CPP_SDK_INSTALL_PREFIX} \ + -DCMAKE_PREFIX_PATH="${YDB_DEPS_PREFIX_PATH}" \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache +RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ + ccache --zero-stats >/dev/null \ + && cmake --build --preset default \ + && cmake --install build \ + && ccache --show-stats + +RUN rm -rf build + RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ cmake --preset ${PRESET} \ + -DYDB_CPP_SDK_SLO_USE_INSTALLED_SDK=ON \ + -DCMAKE_PREFIX_PATH="${YDB_DEPS_PREFIX_PATH}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ diff --git a/tests/slo_workloads/Dockerfile.userver b/tests/slo_workloads/Dockerfile.userver index 737c718140..c14b20bb4e 100644 --- a/tests/slo_workloads/Dockerfile.userver +++ b/tests/slo_workloads/Dockerfile.userver @@ -1,18 +1,11 @@ # syntax=docker/dockerfile:1.7 -# Build userver .deb package (core and other components; YDB is built from source in-repo). -FROM ghcr.io/userver-framework/ubuntu-22.04-userver-base:latest AS userver-deb -ARG USERVER_VERSION=v2.12 -RUN git clone --depth 1 --branch ${USERVER_VERSION} \ - https://github.com/userver-framework/userver.git /userver -WORKDIR /userver -ENV BUILD_OPTIONS="-DUSERVER_FEATURE_YDB=OFF" -RUN ./scripts/build_and_install.sh - -FROM ubuntu:22.04 AS base +FROM ghcr.io/userver-framework/ubuntu-22.04-userver-base:latest AS base ENV DEBIAN_FRONTEND=noninteractive ARG PRESET=release-test-clang +ARG SDK_PRESET=release-clang +ARG USERVER_VERSION=v2.12 # ccache settings consumed by the configure/build steps below. ENV CCACHE_DIR=/root/.ccache @@ -27,6 +20,8 @@ RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \ echo 'Acquire::https::Timeout "60";' >> /etc/apt/apt.conf.d/80-retries ENV WGET_OPTS="--tries=5 --waitretry=15 --timeout=60 --retry-connrefused --retry-on-http-error=500,502,503,504" +ENV YDB_CPP_SDK_INSTALL_PREFIX=/opt/ydb-cpp-sdk +ENV YDB_DEPS_PREFIX_PATH="${YDB_CPP_SDK_INSTALL_PREFIX};/root/ydb_deps/absl;/root/ydb_deps/protobuf;/root/ydb_deps/grpc;/root/ydb_deps/brotli" # Install software-properties-common and add the gcc-13 PPA with shell-level # retry loop — Acquire::Retries doesn't cover TCP connect timeouts. @@ -44,7 +39,7 @@ RUN for i in 1 2 3 4 5; do \ # Install C++ tools and libraries RUN for i in 1 2 3 4 5; do \ apt-get -y install \ - git gdb wget ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ + git gdb wget jq ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \ libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13 && \ break; \ @@ -155,26 +150,39 @@ RUN wget $WGET_OPTS https://github.com/ccache/ccache/releases/download/v${CCACHE && cp ccache-${CCACHE_VERSION}-linux-x86_64/ccache /usr/local/bin/ \ && rm -rf ccache-${CCACHE_VERSION}-linux-x86_64 ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz -FROM base - -ARG PRESET=release-test-clang -ARG USERVER_VERSION=v2.12 - -COPY --from=userver-deb /userver/libuserver-all-dev*.deb /tmp/ -# libuserver-all-dev pulls build-essential (gcc-11), which would override -# gcc-13 alternatives. Reassert gcc-13 after install. -RUN for i in 1 2 3 4 5; do \ +# Prefer the release-attached Debian package. If a matching asset is absent, +# fall back to building the package from the same pinned userver tag. +RUN set -eux; \ + release_json=/tmp/userver-release.json; \ + wget $WGET_OPTS -qO "${release_json}" \ + "https://api.github.com/repos/userver-framework/userver/releases/tags/${USERVER_VERSION}" || touch "${release_json}"; \ + USERVER_DEB_URL="$( \ + jq -r '.assets[]?.browser_download_url // empty' "${release_json}" 2>/dev/null | \ + grep -E 'libuserver-all-dev.*ubuntu.*22\.04.*amd64.*\.deb' | \ + head -n1 || true \ + )"; \ + if [ -n "${USERVER_DEB_URL}" ]; then \ + wget $WGET_OPTS -O /tmp/libuserver-all-dev.deb "${USERVER_DEB_URL}"; \ + else \ + git clone --depth 1 --branch ${USERVER_VERSION} \ + https://github.com/userver-framework/userver.git /tmp/userver-build; \ + cd /tmp/userver-build; \ + BUILD_OPTIONS="-DUSERVER_FEATURE_YDB=OFF" ./scripts/build_and_install.sh; \ + cp libuserver-all-dev*.deb /tmp/libuserver-all-dev.deb; \ + rm -rf /tmp/userver-build; \ + fi; \ + for i in 1 2 3 4 5; do \ apt-get -y update && \ - apt-get -y install --no-install-recommends /tmp/libuserver-all-dev*.deb && \ + apt-get -y install --no-install-recommends /tmp/libuserver-all-dev.deb && \ break; \ echo "userver .deb install attempt $i failed; sleeping $((i * 15))s"; \ sleep $((i * 15)); \ - done \ - && rm -rf /tmp/libuserver-all-dev*.deb /var/lib/apt/lists/* \ - && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000 \ - && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 10000 \ - && update-alternatives --set gcc /usr/bin/gcc-13 \ - && update-alternatives --set g++ /usr/bin/g++-13 + done; \ + rm -rf /tmp/libuserver-all-dev.deb "${release_json}" /var/lib/apt/lists/*; \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000; \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 10000; \ + update-alternatives --set gcc /usr/bin/gcc-13; \ + update-alternatives --set g++ /usr/bin/g++-13 # Userver source checkout for chaotic codegen tools used during configure. RUN for i in 1 2 3 4 5; do \ @@ -193,8 +201,27 @@ COPY . /ydb-cpp-sdk WORKDIR /ydb-cpp-sdk RUN rm -rf build +RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ + cmake --preset ${SDK_PRESET} \ + -DYDB_SDK_INSTALL=ON \ + -DYDB_SDK_EXAMPLES=OFF \ + -DYDB_SDK_TESTS=OFF \ + -DCMAKE_INSTALL_PREFIX=${YDB_CPP_SDK_INSTALL_PREFIX} \ + -DCMAKE_PREFIX_PATH="${YDB_DEPS_PREFIX_PATH}" \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache +RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ + ccache --zero-stats >/dev/null \ + && cmake --build --preset default \ + && cmake --install build \ + && ccache --show-stats + +RUN rm -rf build + RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ cmake --preset ${PRESET} \ + -DYDB_CPP_SDK_SLO_USE_INSTALLED_SDK=ON \ + -DCMAKE_PREFIX_PATH="${YDB_DEPS_PREFIX_PATH}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DYDB_CPP_SDK_BUILD_SLO_USERVER=ON diff --git a/tests/slo_workloads/userver/cmake/SetupYdbCppSDK.cmake b/tests/slo_workloads/userver/cmake/SetupYdbCppSDK.cmake index 0f647dbfce..bbca90b857 100644 --- a/tests/slo_workloads/userver/cmake/SetupYdbCppSDK.cmake +++ b/tests/slo_workloads/userver/cmake/SetupYdbCppSDK.cmake @@ -1,8 +1,8 @@ -# Override userver's SetupYdbCppSDK.cmake: use in-tree YDB-CPP-SDK targets -# from this repo instead of downloading ydb-cpp-sdk via CPM. +# Override userver's SetupYdbCppSDK.cmake: use already configured +# YDB-CPP-SDK targets instead of downloading ydb-cpp-sdk via CPM. if(NOT TARGET YDB-CPP-SDK::Table) - message(FATAL_ERROR "In-tree YDB-CPP-SDK must be configured before building userver-ydb") + message(FATAL_ERROR "YDB-CPP-SDK::Table must be configured before building userver-ydb") endif() set(ydb-cpp-sdk_INCLUDE_DIRS "") diff --git a/tests/unit/client/retry_range/retry_range_ut.cpp b/tests/unit/client/retry_range/retry_range_ut.cpp index 60dd6b8f1c..5062c14e9e 100644 --- a/tests/unit/client/retry_range/retry_range_ut.cpp +++ b/tests/unit/client/retry_range/retry_range_ut.cpp @@ -158,6 +158,16 @@ struct TTableClientFixture { .SetDatabase("/Root/My/DB")); Client = std::make_unique(*Driver); } + + ~TTableClientFixture() { + Client.reset(); + if (Driver) { + Driver->Stop(true); + } + if (GrpcServer) { + GrpcServer->Shutdown(); + } + } }; struct TQueryClientFixture { @@ -180,6 +190,16 @@ struct TQueryClientFixture { .SetDatabase("/Root/My/DB")); Client = std::make_unique(*Driver); } + + ~TQueryClientFixture() { + Client.reset(); + if (Driver) { + Driver->Stop(true); + } + if (GrpcServer) { + GrpcServer->Shutdown(); + } + } }; } // namespace