diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index f905bf7b7812..5b0d6ab6d0de 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -34,7 +34,7 @@ Related: https://github.com/ClickHouse/ClickHouse/pull/NNNNN - [ ] Stateful tests - [ ] Performance tests - [x] Aarch64 tests -- [ ] All with ASAN +- [x] All with ASAN - [x] All with TSAN - [x] All with MSAN - [x] All with UBSAN diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py index f624cc143ac6..32d6a1fc5938 100755 --- a/.github/actions/create_workflow_report/create_workflow_report.py +++ b/.github/actions/create_workflow_report/create_workflow_report.py @@ -311,12 +311,15 @@ def _find_rebase_baseline(branch_ref: str, cwd: str | None) -> str | None: "git", "log", branch_ref, - "--reverse", + "--first-parent", + "-n", + "1", "-i", "-E", "--grep=^Rebase CICD", "--grep=Merge pull request .*rebase-cicd", "--grep=Merge pull request .*from Altinity/rebase/", + "--grep=Merge pull request .*from Altinity/bump/", "--format=%H", ], cwd=cwd, @@ -339,7 +342,7 @@ def get_prs_in_release_dataframe( ) -> pd.DataFrame: f""" PRs merged into branch_ref that belong in the next release notes: after the latest GitHub - Release tag on this history, or after the oldest rebase bootstrap if no such tag exists. + Release tag on this history, or after the latest bootstrap merge if no such tag exists. Only merge commits whose subject has from / (e.g. from Altinity/) are included. Columns: pr_number, pr_name, labels. Omits PRs labeled cicd. """ @@ -356,9 +359,9 @@ def get_prs_in_release_dataframe( check=False, ) - baseline_ref, baseline_sha = _find_release_baseline(branch_ref, repo, cwd) + _, baseline_sha = _find_release_baseline(branch_ref, repo, cwd) if not baseline_sha: - # If no release tag, try to find rebase commit 500 commits back. + # If no release tag, search recent history for the latest branch bootstrap merge. subprocess.run( ["git", "fetch", "--deepen=500", "origin", branch_ref], cwd=cwd, diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 4800d99d0cbb..6d1dc6a0e6c2 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -269,6 +269,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_debug)' --workflow "MasterCI" --ci --timestamp + - name: Upload artifact CH_AMD_DEBUG_GH + id: upload_CH_AMD_DEBUG_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_DEBUG_GH + if: steps.upload_CH_AMD_DEBUG_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_DEBUG_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_asan_ubsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -313,6 +325,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_asan_ubsan)' --workflow "MasterCI" --ci --timestamp + - name: Upload artifact CH_AMD_ASAN_UBSAN_GH + id: upload_CH_AMD_ASAN_UBSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_ASAN_UBSAN_GH + if: steps.upload_CH_AMD_ASAN_UBSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_ASAN_UBSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_binary: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -357,6 +381,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_binary)' --workflow "MasterCI" --ci --timestamp + - name: Upload artifact CH_AMD_BINARY_GH + id: upload_CH_AMD_BINARY_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_BINARY_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_BINARY_GH + if: steps.upload_CH_AMD_BINARY_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_BINARY_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_debug: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -401,6 +437,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_debug)' --workflow "MasterCI" --ci --timestamp + - name: Upload artifact CH_ARM_DEBUG_GH + id: upload_CH_ARM_DEBUG_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_DEBUG_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_DEBUG_GH + if: steps.upload_CH_ARM_DEBUG_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_DEBUG_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_asan_ubsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -445,6 +493,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_asan_ubsan)' --workflow "MasterCI" --ci --timestamp + - name: Upload artifact CH_ARM_ASAN_UBSAN_GH + id: upload_CH_ARM_ASAN_UBSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_ASAN_UBSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_ASAN_UBSAN_GH + if: steps.upload_CH_ARM_ASAN_UBSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_ASAN_UBSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_ubsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -489,6 +549,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_ubsan)' --workflow "MasterCI" --ci --timestamp + - name: Upload artifact CH_ARM_UBSAN_GH + id: upload_CH_ARM_UBSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_UBSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_UBSAN_GH + if: steps.upload_CH_ARM_UBSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_UBSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_binary: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -533,6 +605,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_binary)' --workflow "MasterCI" --ci --timestamp + - name: Upload artifact CH_ARM_BIN_GH + id: upload_CH_ARM_BIN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_BIN_GH + if: steps.upload_CH_ARM_BIN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_BIN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_tsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -577,6 +661,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_tsan)' --workflow "MasterCI" --ci --timestamp + - name: Upload artifact CH_AMD_TSAN_GH + id: upload_CH_AMD_TSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_TSAN_GH + if: steps.upload_CH_AMD_TSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_TSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_msan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -621,6 +717,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_msan)' --workflow "MasterCI" --ci --timestamp + - name: Upload artifact CH_AMD_MSAN_GH + id: upload_CH_AMD_MSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_MSAN_GH + if: steps.upload_CH_AMD_MSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_tsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -665,6 +773,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_tsan)' --workflow "MasterCI" --ci --timestamp + - name: Upload artifact CH_ARM_TSAN_GH + id: upload_CH_ARM_TSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_TSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_TSAN_GH + if: steps.upload_CH_ARM_TSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_TSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_msan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -709,6 +829,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_msan)' --workflow "MasterCI" --ci --timestamp + - name: Upload artifact CH_ARM_MSAN_GH + id: upload_CH_ARM_MSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_MSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_MSAN_GH + if: steps.upload_CH_ARM_MSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_llvm_coverage_per_test: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -1539,6 +1671,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1583,6 +1722,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1627,6 +1773,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1671,6 +1824,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1715,6 +1875,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1759,6 +1926,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1803,6 +1977,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1847,6 +2028,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1891,6 +2079,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1935,6 +2130,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1979,6 +2181,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2023,6 +2232,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2067,6 +2283,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2111,6 +2334,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2155,6 +2385,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2199,6 +2436,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2243,6 +2487,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2287,6 +2538,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2331,6 +2589,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2375,6 +2640,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2419,6 +2691,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2463,6 +2742,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2507,6 +2793,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2551,6 +2844,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2947,17 +3247,75 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_asan_ubsan, azure, parallel)' --workflow "MasterCI" --ci --timestamp - stateless_tests_arm_asan_ubsan_azure_sequential: + stateless_tests_arm_asan_ubsan_azure_sequential_1_2: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64] + needs: [build_arm_asan_ubsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbl91YnNhbiwgYXp1cmUsIHNlcXVlbnRpYWwsIDEvMik=') }} + name: "Stateless tests (arm_asan_ubsan, azure, sequential, 1/2)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (arm_asan_ubsan, azure, sequential, 1/2)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_ARM_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_ASAN_UBSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_asan_ubsan, azure, sequential, 1/2)' --workflow "MasterCI" --ci --timestamp + + stateless_tests_arm_asan_ubsan_azure_sequential_2_2: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64] needs: [build_arm_asan_ubsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbl91YnNhbiwgYXp1cmUsIHNlcXVlbnRpYWwp') }} - name: "Stateless tests (arm_asan_ubsan, azure, sequential)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbl91YnNhbiwgYXp1cmUsIHNlcXVlbnRpYWwsIDIvMik=') }} + name: "Stateless tests (arm_asan_ubsan, azure, sequential, 2/2)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2972,7 +3330,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (arm_asan_ubsan, azure, sequential)" + test_name: "Stateless tests (arm_asan_ubsan, azure, sequential, 2/2)" - name: Prepare env script run: | @@ -2991,11 +3349,18 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_asan_ubsan, azure, sequential)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_asan_ubsan, azure, sequential, 2/2)' --workflow "MasterCI" --ci --timestamp integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_6: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] @@ -3035,6 +3400,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3079,6 +3451,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3123,6 +3502,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3167,6 +3553,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3211,6 +3604,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3255,6 +3655,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3299,6 +3706,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3343,6 +3757,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3387,6 +3808,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3431,6 +3859,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3475,6 +3910,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3519,6 +3961,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3563,6 +4012,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3607,6 +4063,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3651,6 +4114,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3695,6 +4165,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3739,6 +4216,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3783,6 +4267,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3827,6 +4318,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3871,6 +4369,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3915,6 +4420,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3959,6 +4471,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4575,6 +5094,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4619,6 +5145,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4663,6 +5196,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4707,6 +5247,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4751,6 +5298,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4795,6 +5349,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4839,6 +5400,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4883,6 +5451,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -5111,7 +5686,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, clickbench_amd_release, clickbench_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_6, integration_tests_amd_msan_1_6, integration_tests_amd_msan_2_6, integration_tests_amd_msan_3_6, integration_tests_amd_msan_4_6, integration_tests_amd_msan_5_6, integration_tests_amd_msan_6_6, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, clickbench_amd_release, clickbench_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_6, integration_tests_amd_msan_1_6, integration_tests_amd_msan_2_6, integration_tests_amd_msan_3_6, integration_tests_amd_msan_4_6, integration_tests_amd_msan_5_6, integration_tests_amd_msan_6_6, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -5193,7 +5768,7 @@ jobs: workflow_config: ${{ needs.config_workflow.outputs.data }} RegressionTestsAarch64: needs: [config_workflow, build_arm_binary, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.custom_data.ci_exclude_tags, 'regression') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.custom_data.ci_exclude_tags, 'aarch64')}} + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(fromJson(needs.config_workflow.outputs.data).JOB_KV_DATA.ci_exclude_tags, 'regression') && !contains(fromJson(needs.config_workflow.outputs.data).JOB_KV_DATA.ci_exclude_tags, 'aarch64')}} uses: ./.github/workflows/regression.yml secrets: inherit with: @@ -5273,7 +5848,8 @@ jobs: - stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8 - stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8 - stateless_tests_arm_asan_ubsan_azure_parallel - - stateless_tests_arm_asan_ubsan_azure_sequential + - stateless_tests_arm_asan_ubsan_azure_sequential_1_2 + - stateless_tests_arm_asan_ubsan_azure_sequential_2_2 - integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_6 - integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_6 - integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_6 diff --git a/.github/workflows/nightly_keeper.yml b/.github/workflows/nightly_keeper.yml index ef9dd60c7566..46e3b478203b 100644 --- a/.github/workflows/nightly_keeper.yml +++ b/.github/workflows/nightly_keeper.yml @@ -213,6 +213,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_binary)' --workflow "NightlyKeeperNoFaults" --ci --timestamp + - name: Upload artifact CH_ARM_BIN_GH + id: upload_CH_ARM_BIN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_BIN_GH + if: steps.upload_CH_ARM_BIN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_BIN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + keeper_stress_no_faults: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64] needs: [build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm] @@ -250,6 +262,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | diff --git a/.github/workflows/nightly_keeper_faults.yml b/.github/workflows/nightly_keeper_faults.yml index 6b8e3225137a..78e72a5c4f7e 100644 --- a/.github/workflows/nightly_keeper_faults.yml +++ b/.github/workflows/nightly_keeper_faults.yml @@ -213,6 +213,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_binary)' --workflow "NightlyKeeperFaults" --ci --timestamp + - name: Upload artifact CH_ARM_BIN_GH + id: upload_CH_ARM_BIN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_BIN_GH + if: steps.upload_CH_ARM_BIN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_BIN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + keeper_stress_faults: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64] needs: [build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm] @@ -250,6 +262,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 827d7ae53ddb..1ffc01ce1a29 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -390,6 +390,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_debug)' --workflow "PR" --ci --timestamp + - name: Upload artifact CH_AMD_DEBUG_GH + id: upload_CH_AMD_DEBUG_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_DEBUG_GH + if: steps.upload_CH_AMD_DEBUG_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_DEBUG_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_asan_ubsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] @@ -432,6 +444,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_asan_ubsan)' --workflow "PR" --ci --timestamp + - name: Upload artifact CH_AMD_ASAN_UBSAN_GH + id: upload_CH_AMD_ASAN_UBSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_ASAN_UBSAN_GH + if: steps.upload_CH_AMD_ASAN_UBSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_ASAN_UBSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_binary: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] @@ -474,6 +498,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_binary)' --workflow "PR" --ci --timestamp + - name: Upload artifact CH_AMD_BINARY_GH + id: upload_CH_AMD_BINARY_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_BINARY_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_BINARY_GH + if: steps.upload_CH_AMD_BINARY_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_BINARY_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_debug: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] @@ -516,6 +552,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_debug)' --workflow "PR" --ci --timestamp + - name: Upload artifact CH_ARM_DEBUG_GH + id: upload_CH_ARM_DEBUG_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_DEBUG_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_DEBUG_GH + if: steps.upload_CH_ARM_DEBUG_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_DEBUG_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_asan_ubsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] @@ -558,6 +606,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_asan_ubsan)' --workflow "PR" --ci --timestamp + - name: Upload artifact CH_ARM_ASAN_UBSAN_GH + id: upload_CH_ARM_ASAN_UBSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_ASAN_UBSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_ASAN_UBSAN_GH + if: steps.upload_CH_ARM_ASAN_UBSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_ASAN_UBSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_ubsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] @@ -600,6 +660,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_ubsan)' --workflow "PR" --ci --timestamp + - name: Upload artifact CH_ARM_UBSAN_GH + id: upload_CH_ARM_UBSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_UBSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_UBSAN_GH + if: steps.upload_CH_ARM_UBSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_UBSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_binary: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] @@ -642,6 +714,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_binary)' --workflow "PR" --ci --timestamp + - name: Upload artifact CH_ARM_BIN_GH + id: upload_CH_ARM_BIN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_BIN_GH + if: steps.upload_CH_ARM_BIN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_BIN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_tsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] @@ -684,6 +768,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_tsan)' --workflow "PR" --ci --timestamp + - name: Upload artifact CH_AMD_TSAN_GH + id: upload_CH_AMD_TSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_TSAN_GH + if: steps.upload_CH_AMD_TSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_TSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_msan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] @@ -726,6 +822,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_msan)' --workflow "PR" --ci --timestamp + - name: Upload artifact CH_AMD_MSAN_GH + id: upload_CH_AMD_MSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_MSAN_GH + if: steps.upload_CH_AMD_MSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_tsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] @@ -768,6 +876,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_tsan)' --workflow "PR" --ci --timestamp + - name: Upload artifact CH_ARM_TSAN_GH + id: upload_CH_ARM_TSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_TSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_TSAN_GH + if: steps.upload_CH_ARM_TSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_TSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_msan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] @@ -810,6 +930,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_msan)' --workflow "PR" --ci --timestamp + - name: Upload artifact CH_ARM_MSAN_GH + id: upload_CH_ARM_MSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_MSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_MSAN_GH + if: steps.upload_CH_ARM_MSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_release: runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] @@ -930,6 +1062,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -972,6 +1111,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1014,6 +1160,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1056,6 +1209,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1098,6 +1258,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1140,6 +1307,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1182,6 +1356,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1224,6 +1405,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1266,6 +1454,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1308,6 +1503,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1350,6 +1552,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1392,6 +1601,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1434,6 +1650,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1476,6 +1699,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1518,6 +1748,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1560,6 +1797,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1602,6 +1846,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1644,6 +1895,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1686,6 +1944,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1728,6 +1993,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1770,6 +2042,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1812,6 +2091,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1854,6 +2140,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1896,6 +2189,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1938,6 +2238,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1980,6 +2287,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2022,6 +2336,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2064,6 +2385,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2106,6 +2434,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2148,17 +2483,73 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_asan_ubsan, azure, parallel)' --workflow "PR" --ci --timestamp - stateless_tests_arm_asan_ubsan_azure_sequential: + stateless_tests_arm_asan_ubsan_azure_sequential_1_2: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64] + needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbl91YnNhbiwgYXp1cmUsIHNlcXVlbnRpYWwsIDEvMik=') }} + name: "Stateless tests (arm_asan_ubsan, azure, sequential, 1/2)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (arm_asan_ubsan, azure, sequential, 1/2)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_ARM_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_ASAN_UBSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_asan_ubsan, azure, sequential, 1/2)' --workflow "PR" --ci --timestamp + + stateless_tests_arm_asan_ubsan_azure_sequential_2_2: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbl91YnNhbiwgYXp1cmUsIHNlcXVlbnRpYWwp') }} - name: "Stateless tests (arm_asan_ubsan, azure, sequential)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbl91YnNhbiwgYXp1cmUsIHNlcXVlbnRpYWwsIDIvMik=') }} + name: "Stateless tests (arm_asan_ubsan, azure, sequential, 2/2)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2173,7 +2564,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (arm_asan_ubsan, azure, sequential)" + test_name: "Stateless tests (arm_asan_ubsan, azure, sequential, 2/2)" - name: Prepare env script run: | @@ -2190,11 +2581,18 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_asan_ubsan, azure, sequential)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_asan_ubsan, azure, sequential, 2/2)' --workflow "PR" --ci --timestamp integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_6: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] @@ -2232,6 +2630,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2274,6 +2679,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2316,6 +2728,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2358,6 +2777,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2400,6 +2826,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2442,6 +2875,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2484,6 +2924,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2526,6 +2973,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2568,6 +3022,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2610,6 +3071,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2652,6 +3120,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2694,6 +3169,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2736,6 +3218,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2778,6 +3267,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2820,6 +3316,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2862,6 +3365,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2904,6 +3414,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2946,6 +3463,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -2988,6 +3512,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3030,6 +3561,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3072,6 +3610,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -3114,6 +3659,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4122,6 +4674,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4164,6 +4723,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4206,6 +4772,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4248,6 +4821,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4290,6 +4870,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_DEBUG_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4332,6 +4919,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_ASAN_UBSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4374,6 +4968,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4416,6 +5017,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4542,6 +5150,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -4678,7 +5293,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tidy, build_arm_tsan, build_arm_ubsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_6, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_1_6, integration_tests_amd_msan_2_6, integration_tests_amd_msan_3_6, integration_tests_amd_msan_4_6, integration_tests_amd_msan_5_6, integration_tests_amd_msan_6_6, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tidy, build_arm_tsan, build_arm_ubsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_6, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_6, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_1_6, integration_tests_amd_msan_2_6, integration_tests_amd_msan_3_6, integration_tests_amd_msan_4_6, integration_tests_amd_msan_5_6, integration_tests_amd_msan_6_6, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -4758,7 +5373,7 @@ jobs: workflow_config: ${{ needs.config_workflow.outputs.data }} RegressionTestsAarch64: needs: [config_workflow, build_arm_binary, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.custom_data.ci_exclude_tags, 'regression') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.custom_data.ci_exclude_tags, 'aarch64')}} + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(fromJson(needs.config_workflow.outputs.data).JOB_KV_DATA.ci_exclude_tags, 'regression') && !contains(fromJson(needs.config_workflow.outputs.data).JOB_KV_DATA.ci_exclude_tags, 'aarch64')}} uses: ./.github/workflows/regression.yml secrets: inherit with: @@ -4822,7 +5437,8 @@ jobs: - stateless_tests_arm_binary_parallel - stateless_tests_arm_binary_sequential - stateless_tests_arm_asan_ubsan_azure_parallel - - stateless_tests_arm_asan_ubsan_azure_sequential + - stateless_tests_arm_asan_ubsan_azure_sequential_1_2 + - stateless_tests_arm_asan_ubsan_azure_sequential_2_2 - integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_6 - integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_6 - integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_6 diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index 5e85fe14962b..04fe68b800db 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -166,14 +166,14 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_debug)' --workflow "Community PR" --ci --timestamp - name: Upload artifact CH_AMD_DEBUG - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: CH_AMD_DEBUG path: ci/tmp/build/programs/self-extracting/clickhouse - name: Upload artifact DEB_AMD_DEBUG - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: DEB_AMD_DEBUG path: ci/tmp/*.deb @@ -221,21 +221,21 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_asan_ubsan)' --workflow "Community PR" --ci --timestamp - name: Upload artifact UNITTEST_AMD_ASAN_UBSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: UNITTEST_AMD_ASAN_UBSAN path: ci/tmp/build/src/unit_tests_dbms - name: Upload artifact CH_AMD_ASAN_UBSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: CH_AMD_ASAN_UBSAN path: ci/tmp/build/programs/self-extracting/clickhouse - name: Upload artifact DEB_AMD_ASAN_UBSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: DEB_AMD_ASAN_UBSAN path: ci/tmp/*.deb @@ -283,7 +283,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_binary)' --workflow "Community PR" --ci --timestamp - name: Upload artifact CH_AMD_BINARY - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: CH_AMD_BINARY path: ci/tmp/build/programs/self-extracting/clickhouse @@ -331,14 +331,14 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_debug)' --workflow "Community PR" --ci --timestamp - name: Upload artifact CH_ARM_DEBUG - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: CH_ARM_DEBUG path: ci/tmp/build/programs/self-extracting/clickhouse - name: Upload artifact DEB_ARM_DEBUG - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: DEB_ARM_DEBUG path: ci/tmp/*.deb @@ -386,14 +386,14 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_asan_ubsan)' --workflow "Community PR" --ci --timestamp - name: Upload artifact CH_ARM_ASAN_UBSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: CH_ARM_ASAN_UBSAN path: ci/tmp/build/programs/self-extracting/clickhouse - name: Upload artifact DEB_ARM_ASAN_UBSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: DEB_ARM_ASAN_UBSAN path: ci/tmp/*.deb @@ -441,14 +441,14 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_ubsan)' --workflow "Community PR" --ci --timestamp - name: Upload artifact CH_ARM_UBSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: CH_ARM_UBSAN path: ci/tmp/build/programs/self-extracting/clickhouse - name: Upload artifact DEB_ARM_UBSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: DEB_ARM_UBSAN path: ci/tmp/*.deb @@ -496,7 +496,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_binary)' --workflow "Community PR" --ci --timestamp - name: Upload artifact CH_ARM_BIN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: CH_ARM_BIN path: ci/tmp/build/programs/self-extracting/clickhouse @@ -544,21 +544,21 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_tsan)' --workflow "Community PR" --ci --timestamp - name: Upload artifact UNITTEST_AMD_TSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: UNITTEST_AMD_TSAN path: ci/tmp/build/src/unit_tests_dbms - name: Upload artifact CH_AMD_TSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: CH_AMD_TSAN path: ci/tmp/build/programs/self-extracting/clickhouse - name: Upload artifact DEB_AMD_TSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: DEB_AMD_TSAN path: ci/tmp/*.deb @@ -606,21 +606,21 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_msan)' --workflow "Community PR" --ci --timestamp - name: Upload artifact UNITTEST_AMD_MSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: UNITTEST_AMD_MSAN path: ci/tmp/build/src/unit_tests_dbms - name: Upload artifact CH_AMD_MSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: CH_AMD_MSAN path: ci/tmp/build/programs/self-extracting/clickhouse - name: Upload artifact DEB_AMD_MSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: DEB_AMD_MSAN path: ci/tmp/*.deb @@ -668,14 +668,14 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_tsan)' --workflow "Community PR" --ci --timestamp - name: Upload artifact CH_ARM_TSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: CH_ARM_TSAN path: ci/tmp/build/programs/self-extracting/clickhouse - name: Upload artifact DEB_ARM_TSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: DEB_ARM_TSAN path: ci/tmp/*.deb @@ -723,14 +723,14 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_msan)' --workflow "Community PR" --ci --timestamp - name: Upload artifact CH_ARM_MSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: CH_ARM_MSAN path: ci/tmp/build/programs/self-extracting/clickhouse - name: Upload artifact DEB_ARM_MSAN - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: DEB_ARM_MSAN path: ci/tmp/*.deb @@ -778,28 +778,28 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "Community PR" --ci --timestamp - name: Upload artifact CH_AMD_RELEASE - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: CH_AMD_RELEASE path: ci/tmp/build/programs/self-extracting/clickhouse - name: Upload artifact DEB_AMD_RELEASE - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: DEB_AMD_RELEASE path: ci/tmp/*.deb - name: Upload artifact RPM_AMD_RELEASE - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: RPM_AMD_RELEASE path: ci/tmp/*.rpm - name: Upload artifact TGZ_AMD_RELEASE - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: TGZ_AMD_RELEASE path: ci/tmp/*64.tgz* @@ -847,28 +847,28 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_release)' --workflow "Community PR" --ci --timestamp - name: Upload artifact CH_ARM_RELEASE - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: CH_ARM_RELEASE path: ci/tmp/build/programs/self-extracting/clickhouse - name: Upload artifact DEB_ARM_RELEASE - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: DEB_ARM_RELEASE path: ci/tmp/*.deb - name: Upload artifact RPM_ARM_RELEASE - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: RPM_ARM_RELEASE path: ci/tmp/*.rpm - name: Upload artifact TGZ_ARM_RELEASE - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: TGZ_ARM_RELEASE path: ci/tmp/*64.tgz* @@ -910,7 +910,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_DEBUG - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_DEBUG path: ./ci/tmp @@ -958,7 +958,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_ASAN_UBSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_ASAN_UBSAN path: ./ci/tmp @@ -1006,7 +1006,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_ASAN_UBSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_ASAN_UBSAN path: ./ci/tmp @@ -1054,7 +1054,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_ASAN_UBSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_ASAN_UBSAN path: ./ci/tmp @@ -1102,7 +1102,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_ASAN_UBSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_ASAN_UBSAN path: ./ci/tmp @@ -1150,7 +1150,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_DEBUG - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_DEBUG path: ./ci/tmp @@ -1198,7 +1198,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_DEBUG - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_DEBUG path: ./ci/tmp @@ -1246,7 +1246,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_TSAN path: ./ci/tmp @@ -1294,7 +1294,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_TSAN path: ./ci/tmp @@ -1342,7 +1342,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_TSAN path: ./ci/tmp @@ -1390,7 +1390,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_TSAN path: ./ci/tmp @@ -1438,7 +1438,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_MSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_MSAN path: ./ci/tmp @@ -1486,7 +1486,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_MSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_MSAN path: ./ci/tmp @@ -1534,7 +1534,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_MSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_MSAN path: ./ci/tmp @@ -1582,7 +1582,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_MSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_MSAN path: ./ci/tmp @@ -1630,7 +1630,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_MSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_MSAN path: ./ci/tmp @@ -1678,7 +1678,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_MSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_MSAN path: ./ci/tmp @@ -1726,7 +1726,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_DEBUG - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_DEBUG path: ./ci/tmp @@ -1774,7 +1774,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_DEBUG - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_DEBUG path: ./ci/tmp @@ -1822,7 +1822,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_TSAN path: ./ci/tmp @@ -1870,7 +1870,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_TSAN path: ./ci/tmp @@ -1918,7 +1918,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_TSAN path: ./ci/tmp @@ -1966,7 +1966,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_TSAN path: ./ci/tmp @@ -2014,7 +2014,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_ARM_BIN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_ARM_BIN path: ./ci/tmp @@ -2062,7 +2062,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_ARM_BIN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_ARM_BIN path: ./ci/tmp @@ -2110,7 +2110,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_ASAN_UBSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_ASAN_UBSAN path: ./ci/tmp @@ -2158,7 +2158,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_ASAN_UBSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_ASAN_UBSAN path: ./ci/tmp @@ -2206,7 +2206,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_ASAN_UBSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_ASAN_UBSAN path: ./ci/tmp @@ -2254,7 +2254,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_ASAN_UBSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_ASAN_UBSAN path: ./ci/tmp @@ -2302,7 +2302,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_ASAN_UBSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_ASAN_UBSAN path: ./ci/tmp @@ -2350,7 +2350,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_ASAN_UBSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_ASAN_UBSAN path: ./ci/tmp @@ -2398,7 +2398,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_ARM_BIN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_ARM_BIN path: ./ci/tmp @@ -2446,7 +2446,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_ARM_BIN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_ARM_BIN path: ./ci/tmp @@ -2494,7 +2494,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_ARM_BIN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_ARM_BIN path: ./ci/tmp @@ -2542,7 +2542,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_ARM_BIN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_ARM_BIN path: ./ci/tmp @@ -2590,7 +2590,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_TSAN path: ./ci/tmp @@ -2638,7 +2638,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_TSAN path: ./ci/tmp @@ -2686,7 +2686,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_TSAN path: ./ci/tmp @@ -2734,7 +2734,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_TSAN path: ./ci/tmp @@ -2782,7 +2782,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_TSAN path: ./ci/tmp @@ -2830,7 +2830,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_TSAN path: ./ci/tmp @@ -2878,7 +2878,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_MSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_MSAN path: ./ci/tmp @@ -2926,7 +2926,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_MSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_MSAN path: ./ci/tmp @@ -2974,7 +2974,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_MSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_MSAN path: ./ci/tmp @@ -3022,7 +3022,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_MSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_MSAN path: ./ci/tmp @@ -3070,7 +3070,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_MSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_MSAN path: ./ci/tmp @@ -3118,7 +3118,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_MSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_MSAN path: ./ci/tmp @@ -3166,7 +3166,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact UNITTEST_AMD_ASAN_UBSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: UNITTEST_AMD_ASAN_UBSAN path: ./ci/tmp @@ -3214,7 +3214,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact UNITTEST_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: UNITTEST_AMD_TSAN path: ./ci/tmp @@ -3262,7 +3262,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact UNITTEST_AMD_MSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: UNITTEST_AMD_MSAN path: ./ci/tmp @@ -3310,7 +3310,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact UNITTEST_AMD_ASAN_UBSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: UNITTEST_AMD_ASAN_UBSAN path: ./ci/tmp @@ -3358,7 +3358,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact UNITTEST_AMD_TSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: UNITTEST_AMD_TSAN path: ./ci/tmp @@ -3406,7 +3406,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact UNITTEST_AMD_MSAN - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: UNITTEST_AMD_MSAN path: ./ci/tmp @@ -3454,28 +3454,28 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_AMD_RELEASE - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_AMD_RELEASE path: ./ci/tmp - name: Download artifact DEB_AMD_RELEASE - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: DEB_AMD_RELEASE path: ./ci/tmp - name: Download artifact RPM_AMD_RELEASE - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: RPM_AMD_RELEASE path: ./ci/tmp - name: Download artifact TGZ_AMD_RELEASE - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: TGZ_AMD_RELEASE path: ./ci/tmp @@ -3523,28 +3523,28 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact CH_ARM_RELEASE - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: CH_ARM_RELEASE path: ./ci/tmp - name: Download artifact DEB_ARM_RELEASE - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: DEB_ARM_RELEASE path: ./ci/tmp - name: Download artifact RPM_ARM_RELEASE - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: RPM_ARM_RELEASE path: ./ci/tmp - name: Download artifact TGZ_ARM_RELEASE - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: TGZ_ARM_RELEASE path: ./ci/tmp @@ -3592,7 +3592,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact DEB_AMD_RELEASE - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: DEB_AMD_RELEASE path: ./ci/tmp @@ -3640,7 +3640,7 @@ jobs: ENV_SETUP_SCRIPT_EOF - name: Download artifact DEB_ARM_RELEASE - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: DEB_ARM_RELEASE path: ./ci/tmp diff --git a/.github/workflows/release_builds.yml b/.github/workflows/release_builds.yml index 7d30ba671385..2e2cb30ee34a 100644 --- a/.github/workflows/release_builds.yml +++ b/.github/workflows/release_builds.yml @@ -264,6 +264,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_debug)' --workflow "Release Builds" --ci --timestamp + - name: Upload artifact CH_AMD_DEBUG_GH + id: upload_CH_AMD_DEBUG_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_DEBUG_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_DEBUG_GH + if: steps.upload_CH_AMD_DEBUG_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_DEBUG_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_asan_ubsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -308,6 +320,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_asan_ubsan)' --workflow "Release Builds" --ci --timestamp + - name: Upload artifact CH_AMD_ASAN_UBSAN_GH + id: upload_CH_AMD_ASAN_UBSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_ASAN_UBSAN_GH + if: steps.upload_CH_AMD_ASAN_UBSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_ASAN_UBSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_binary: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -352,6 +376,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_binary)' --workflow "Release Builds" --ci --timestamp + - name: Upload artifact CH_AMD_BINARY_GH + id: upload_CH_AMD_BINARY_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_BINARY_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_BINARY_GH + if: steps.upload_CH_AMD_BINARY_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_BINARY_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_debug: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -396,6 +432,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_debug)' --workflow "Release Builds" --ci --timestamp + - name: Upload artifact CH_ARM_DEBUG_GH + id: upload_CH_ARM_DEBUG_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_DEBUG_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_DEBUG_GH + if: steps.upload_CH_ARM_DEBUG_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_DEBUG_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_asan_ubsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -440,6 +488,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_asan_ubsan)' --workflow "Release Builds" --ci --timestamp + - name: Upload artifact CH_ARM_ASAN_UBSAN_GH + id: upload_CH_ARM_ASAN_UBSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_ASAN_UBSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_ASAN_UBSAN_GH + if: steps.upload_CH_ARM_ASAN_UBSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_ASAN_UBSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_ubsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -484,6 +544,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_ubsan)' --workflow "Release Builds" --ci --timestamp + - name: Upload artifact CH_ARM_UBSAN_GH + id: upload_CH_ARM_UBSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_UBSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_UBSAN_GH + if: steps.upload_CH_ARM_UBSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_UBSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_binary: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -528,6 +600,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_binary)' --workflow "Release Builds" --ci --timestamp + - name: Upload artifact CH_ARM_BIN_GH + id: upload_CH_ARM_BIN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_BIN_GH + if: steps.upload_CH_ARM_BIN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_BIN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_tsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -572,6 +656,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_tsan)' --workflow "Release Builds" --ci --timestamp + - name: Upload artifact CH_AMD_TSAN_GH + id: upload_CH_AMD_TSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_TSAN_GH + if: steps.upload_CH_AMD_TSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_TSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_msan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -616,6 +712,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_msan)' --workflow "Release Builds" --ci --timestamp + - name: Upload artifact CH_AMD_MSAN_GH + id: upload_CH_AMD_MSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_AMD_MSAN_GH + if: steps.upload_CH_AMD_MSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_tsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -660,6 +768,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_tsan)' --workflow "Release Builds" --ci --timestamp + - name: Upload artifact CH_ARM_TSAN_GH + id: upload_CH_ARM_TSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_TSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_TSAN_GH + if: steps.upload_CH_ARM_TSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_TSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_arm_msan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -704,6 +824,18 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_msan)' --workflow "Release Builds" --ci --timestamp + - name: Upload artifact CH_ARM_MSAN_GH + id: upload_CH_ARM_MSAN_GH + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: CH_ARM_MSAN_GH + path: ci/tmp/build/programs/self-extracting/clickhouse + retention-days: 1 + - name: Warn on failed upload of CH_ARM_MSAN_GH + if: steps.upload_CH_ARM_MSAN_GH.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." + build_amd_release: runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -1138,6 +1270,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | @@ -1182,6 +1321,13 @@ jobs: EOF ENV_SETUP_SCRIPT_EOF + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + - name: Run id: run run: | diff --git a/ci/defs/defs.py b/ci/defs/defs.py index 6ebc34260aff..663e3b8f2d3c 100644 --- a/ci/defs/defs.py +++ b/ci/defs/defs.py @@ -451,6 +451,20 @@ class ArtifactNames: CH_RISCV64 = "CH_RISCV64_BIN" CH_S390X = "CH_S390X_BIN" CH_LOONGARCH64 = "CH_LOONGARCH64_BIN" + CH_AMD_DEBUG_GH = "CH_AMD_DEBUG_GH" + CH_ARM_DEBUG_GH = "CH_ARM_DEBUG_GH" + CH_AMD_BINARY_GH = "CH_AMD_BINARY_GH" + CH_ARM_BINARY_GH = "CH_ARM_BIN_GH" + CH_AMD_ASAN_GH = "CH_AMD_ASAN_GH" + CH_AMD_TSAN_GH = "CH_AMD_TSAN_GH" + CH_AMD_MSAN_GH = "CH_AMD_MSAN_GH" + CH_AMD_UBSAN_GH = "CH_AMD_UBSAN_GH" + CH_AMD_ASAN_UBSAN_GH = "CH_AMD_ASAN_UBSAN_GH" + CH_ARM_ASAN_GH = "CH_ARM_ASAN_GH" + CH_ARM_TSAN_GH = "CH_ARM_TSAN_GH" + CH_ARM_UBSAN_GH = "CH_ARM_UBSAN_GH" + CH_ARM_ASAN_UBSAN_GH = "CH_ARM_ASAN_UBSAN_GH" + CH_ARM_MSAN_GH = "CH_ARM_MSAN_GH" FAST_TEST = "FAST_TEST" UNITTEST_AMD_ASAN_UBSAN = "UNITTEST_AMD_ASAN_UBSAN" @@ -572,6 +586,28 @@ class ArtifactConfigs: ArtifactNames.CH_LOONGARCH64, ] ) + clickhouse_binaries_gh = Artifact.Config( + name="...", + type=Artifact.Type.GH, + path=f"{TEMP_DIR}/build/programs/self-extracting/clickhouse", + ).parametrize( + names=[ + ArtifactNames.CH_AMD_DEBUG_GH, + ArtifactNames.CH_AMD_BINARY_GH, + ArtifactNames.CH_ARM_BINARY_GH, + ArtifactNames.CH_AMD_ASAN_GH, + ArtifactNames.CH_AMD_TSAN_GH, + ArtifactNames.CH_AMD_MSAN_GH, + ArtifactNames.CH_AMD_UBSAN_GH, + ArtifactNames.CH_ARM_ASAN_GH, + ArtifactNames.CH_ARM_TSAN_GH, + ArtifactNames.CH_AMD_ASAN_UBSAN_GH, + ArtifactNames.CH_ARM_ASAN_UBSAN_GH, + ArtifactNames.CH_ARM_UBSAN_GH, + ArtifactNames.CH_ARM_MSAN_GH, + ArtifactNames.CH_ARM_DEBUG_GH, + ] + ) clickhouse_stripped_binaries = Artifact.Config( name="...", type=Artifact.Type.S3, diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index 6311483e3779..f6ae41f58278 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -283,13 +283,18 @@ class JobConfigs: ).parametrize( Job.ParamSet( parameter=BuildTypes.AMD_DEBUG, - provides=[ArtifactNames.CH_AMD_DEBUG, ArtifactNames.DEB_AMD_DEBUG], + provides=[ + ArtifactNames.CH_AMD_DEBUG, + ArtifactNames.CH_AMD_DEBUG_GH, + ArtifactNames.DEB_AMD_DEBUG, + ], runs_on=RunnerLabels.FUNC_TESTER_AMD, ), Job.ParamSet( parameter=BuildTypes.AMD_ASAN_UBSAN, provides=[ ArtifactNames.CH_AMD_ASAN_UBSAN, + ArtifactNames.CH_AMD_ASAN_UBSAN_GH, ArtifactNames.DEB_AMD_ASAN_UBSAN, ArtifactNames.UNITTEST_AMD_ASAN_UBSAN, ], @@ -297,30 +302,45 @@ class JobConfigs: ), Job.ParamSet( parameter=BuildTypes.AMD_BINARY, - provides=[ArtifactNames.CH_AMD_BINARY], + provides=[ + ArtifactNames.CH_AMD_BINARY, + ArtifactNames.CH_AMD_BINARY_GH, + ], runs_on=RunnerLabels.FUNC_TESTER_AMD, ), Job.ParamSet( parameter=BuildTypes.ARM_DEBUG, - provides=[ArtifactNames.CH_ARM_DEBUG, ArtifactNames.DEB_ARM_DEBUG], + provides=[ + ArtifactNames.CH_ARM_DEBUG, + ArtifactNames.CH_ARM_DEBUG_GH, + ArtifactNames.DEB_ARM_DEBUG, + ], runs_on=RunnerLabels.FUNC_TESTER_AMD, ), Job.ParamSet( parameter=BuildTypes.ARM_ASAN_UBSAN, provides=[ ArtifactNames.CH_ARM_ASAN_UBSAN, + ArtifactNames.CH_ARM_ASAN_UBSAN_GH, ArtifactNames.DEB_ARM_ASAN_UBSAN, ], runs_on=RunnerLabels.FUNC_TESTER_AMD, ), Job.ParamSet( parameter=BuildTypes.ARM_UBSAN, - provides=[ArtifactNames.CH_ARM_UBSAN, ArtifactNames.DEB_ARM_UBSAN], + provides=[ + ArtifactNames.CH_ARM_UBSAN, + ArtifactNames.CH_ARM_UBSAN_GH, + ArtifactNames.DEB_ARM_UBSAN, + ], runs_on=RunnerLabels.FUNC_TESTER_AMD, ), Job.ParamSet( parameter=BuildTypes.ARM_BINARY, - provides=[ArtifactNames.CH_ARM_BINARY], + provides=[ + ArtifactNames.CH_ARM_BINARY, + ArtifactNames.CH_ARM_BINARY_GH, + ], runs_on=RunnerLabels.FUNC_TESTER_AMD, ), ) + common_sanitizer_build_job_config.set_post_hooks( @@ -333,6 +353,7 @@ class JobConfigs: parameter=BuildTypes.AMD_TSAN, provides=[ ArtifactNames.CH_AMD_TSAN, + ArtifactNames.CH_AMD_TSAN_GH, ArtifactNames.DEB_AMD_TSAN, ArtifactNames.UNITTEST_AMD_TSAN, ], @@ -342,6 +363,7 @@ class JobConfigs: parameter=BuildTypes.AMD_MSAN, provides=[ ArtifactNames.CH_AMD_MSAN, + ArtifactNames.CH_AMD_MSAN_GH, ArtifactNames.DEB_AMD_MSAN, ArtifactNames.UNITTEST_AMD_MSAN, ], @@ -351,13 +373,18 @@ class JobConfigs: parameter=BuildTypes.ARM_TSAN, provides=[ ArtifactNames.CH_ARM_TSAN, + ArtifactNames.CH_ARM_TSAN_GH, ArtifactNames.DEB_ARM_TSAN, ], runs_on=RunnerLabels.FUNC_TESTER_AMD, ), Job.ParamSet( parameter=BuildTypes.ARM_MSAN, - provides=[ArtifactNames.CH_ARM_MSAN, ArtifactNames.DEB_ARM_MSAN], + provides=[ + ArtifactNames.CH_ARM_MSAN, + ArtifactNames.CH_ARM_MSAN_GH, + ArtifactNames.DEB_ARM_MSAN, + ], runs_on=RunnerLabels.FUNC_TESTER_AMD, ), ) @@ -553,29 +580,29 @@ class JobConfigs: Job.ParamSet( parameter="amd_asan_ubsan, flaky check", runs_on=RunnerLabels.AMD_MEDIUM, - requires=[ArtifactNames.CH_AMD_ASAN_UBSAN], + requires=[ArtifactNames.CH_AMD_ASAN_UBSAN_GH], ), Job.ParamSet( parameter="amd_tsan, flaky check", runs_on=RunnerLabels.AMD_LARGE, - requires=[ArtifactNames.CH_AMD_TSAN], + requires=[ArtifactNames.CH_AMD_TSAN_GH], ), Job.ParamSet( parameter="amd_msan, flaky check", runs_on=RunnerLabels.AMD_LARGE, - requires=[ArtifactNames.CH_AMD_MSAN], + requires=[ArtifactNames.CH_AMD_MSAN_GH], ), Job.ParamSet( parameter="amd_debug, flaky check", runs_on=RunnerLabels.AMD_MEDIUM, - requires=[ArtifactNames.CH_AMD_DEBUG], + requires=[ArtifactNames.CH_AMD_DEBUG_GH], ), ) stateless_tests_targeted_pr_jobs = common_ft_job_config.parametrize( Job.ParamSet( parameter="arm_asan_ubsan, targeted", runs_on=RunnerLabels.ARM_LARGE, - requires=[ArtifactNames.CH_ARM_ASAN_UBSAN], + requires=[ArtifactNames.CH_ARM_ASAN_UBSAN_GH], ), ) # --root/--privileged/--cgroupns=host is required for clickhouse-test --memory-limit @@ -608,7 +635,7 @@ class JobConfigs: "./ci/jobs/queries", ], ), - requires=[ArtifactNames.CH_AMD_DEBUG], + requires=[ArtifactNames.CH_AMD_DEBUG_GH], runs_on=RunnerLabels.AMD_SMALL, ) functional_tests_jobs = common_ft_job_config.parametrize( @@ -616,7 +643,7 @@ class JobConfigs: Job.ParamSet( parameter=f"amd_asan_ubsan, distributed plan, parallel, {batch}/{total_batches}", runs_on=RunnerLabels.AMD_MEDIUM_CPU, - requires=[ArtifactNames.CH_AMD_ASAN_UBSAN], + requires=[ArtifactNames.CH_AMD_ASAN_UBSAN_GH], ) for total_batches in (2,) for batch in range(1, total_batches + 1) @@ -625,7 +652,7 @@ class JobConfigs: Job.ParamSet( parameter=f"amd_asan_ubsan, db disk, distributed plan, sequential, {batch}/{total_batches}", runs_on=RunnerLabels.AMD_SMALL_MEM, - requires=[ArtifactNames.CH_AMD_ASAN_UBSAN], + requires=[ArtifactNames.CH_AMD_ASAN_UBSAN_GH], ) for total_batches in (2,) for batch in range(1, total_batches + 1) @@ -669,18 +696,18 @@ class JobConfigs: Job.ParamSet( parameter="amd_debug, parallel", runs_on=RunnerLabels.AMD_MEDIUM_CPU, - requires=[ArtifactNames.CH_AMD_DEBUG], + requires=[ArtifactNames.CH_AMD_DEBUG_GH], ), Job.ParamSet( parameter="amd_debug, sequential", runs_on=RunnerLabels.FUNC_TESTER_AMD, - requires=[ArtifactNames.CH_AMD_DEBUG], + requires=[ArtifactNames.CH_AMD_DEBUG_GH], ), *[ Job.ParamSet( parameter=f"amd_tsan, parallel, {batch}/{total_batches}", runs_on=RunnerLabels.AMD_LARGE, - requires=[ArtifactNames.CH_AMD_TSAN], + requires=[ArtifactNames.CH_AMD_TSAN_GH], ) for total_batches in (2,) for batch in range(1, total_batches + 1) @@ -689,7 +716,7 @@ class JobConfigs: Job.ParamSet( parameter=f"amd_tsan, sequential, {batch}/{total_batches}", runs_on=RunnerLabels.FUNC_TESTER_AMD, - requires=[ArtifactNames.CH_AMD_TSAN], + requires=[ArtifactNames.CH_AMD_TSAN_GH], ) for total_batches in (2,) for batch in range(1, total_batches + 1) @@ -698,7 +725,7 @@ class JobConfigs: Job.ParamSet( parameter=f"amd_msan, WasmEdge, parallel, {batch}/{total_batches}", runs_on=RunnerLabels.FUNC_TESTER_AMD, - requires=[ArtifactNames.CH_AMD_MSAN], + requires=[ArtifactNames.CH_AMD_MSAN_GH], ) for total_batches in (4,) for batch in range(1, total_batches + 1) @@ -707,7 +734,7 @@ class JobConfigs: Job.ParamSet( parameter=f"amd_msan, WasmEdge, sequential, {batch}/{total_batches}", runs_on=RunnerLabels.FUNC_TESTER_AMD, - requires=[ArtifactNames.CH_AMD_MSAN], + requires=[ArtifactNames.CH_AMD_MSAN_GH], ) for total_batches in (2,) for batch in range(1, total_batches + 1) @@ -715,18 +742,18 @@ class JobConfigs: Job.ParamSet( parameter="amd_debug, distributed plan, s3 storage, parallel", runs_on=RunnerLabels.AMD_MEDIUM, # large machine - no boost, why? - requires=[ArtifactNames.CH_AMD_DEBUG], + requires=[ArtifactNames.CH_AMD_DEBUG_GH], ), Job.ParamSet( parameter="amd_debug, distributed plan, s3 storage, sequential", runs_on=RunnerLabels.FUNC_TESTER_AMD, - requires=[ArtifactNames.CH_AMD_DEBUG], + requires=[ArtifactNames.CH_AMD_DEBUG_GH], ), *[ Job.ParamSet( parameter=f"amd_tsan, s3 storage, parallel, {batch}/{total_batches}", runs_on=RunnerLabels.AMD_MEDIUM, - requires=[ArtifactNames.CH_AMD_TSAN], + requires=[ArtifactNames.CH_AMD_TSAN_GH], ) for total_batches in (2,) for batch in range(1, total_batches + 1) @@ -735,7 +762,7 @@ class JobConfigs: Job.ParamSet( parameter=f"amd_tsan, s3 storage, sequential, {batch}/{total_batches}", runs_on=RunnerLabels.FUNC_TESTER_AMD, - requires=[ArtifactNames.CH_AMD_TSAN], + requires=[ArtifactNames.CH_AMD_TSAN_GH], ) for total_batches in (2,) for batch in range(1, total_batches + 1) @@ -743,12 +770,12 @@ class JobConfigs: Job.ParamSet( parameter="arm_binary, parallel", runs_on=RunnerLabels.ARM_MEDIUM_CPU, - requires=[ArtifactNames.CH_ARM_BINARY], + requires=[ArtifactNames.CH_ARM_BINARY_GH], ), Job.ParamSet( parameter="arm_binary, sequential", runs_on=RunnerLabels.FUNC_TESTER_ARM, - requires=[ArtifactNames.CH_ARM_BINARY], + requires=[ArtifactNames.CH_ARM_BINARY_GH], ), ) functional_tests_jobs_coverage = common_ft_job_config.parametrize( @@ -768,13 +795,18 @@ class JobConfigs: Job.ParamSet( parameter="arm_asan_ubsan, azure, parallel", runs_on=RunnerLabels.ARM_MEDIUM, - requires=[ArtifactNames.CH_ARM_ASAN_UBSAN], - ), - Job.ParamSet( - parameter="arm_asan_ubsan, azure, sequential", - runs_on=RunnerLabels.ARM_SMALL_MEM, - requires=[ArtifactNames.CH_ARM_ASAN_UBSAN], + requires=[ArtifactNames.CH_ARM_ASAN_UBSAN_GH], ), + *[ + Job.ParamSet( + parameter=f"arm_asan_ubsan, azure, sequential, {batch}/{total_batches}", + runs_on=RunnerLabels.FUNC_TESTER_ARM, + requires=[ArtifactNames.CH_ARM_ASAN_UBSAN_GH], + timeout=3600 * 4, + ) + for total_batches in (2,) + for batch in range(1, total_batches + 1) + ], ) bugfix_validation_it_job = ( common_integration_test_job_config.set_name(JobNames.BUGFIX_VALIDATE_IT) @@ -925,7 +957,7 @@ class JobConfigs: Job.ParamSet( parameter=f"amd_asan_ubsan, db disk, {batch}/{total_batches}", runs_on=RunnerLabels.AMD_MEDIUM, - requires=[ArtifactNames.CH_AMD_ASAN_UBSAN], + requires=[ArtifactNames.CH_AMD_ASAN_UBSAN_GH], ) for total_batches in (4,) for batch in range(1, total_batches + 1) @@ -936,7 +968,7 @@ class JobConfigs: Job.ParamSet( parameter=f"amd_asan_ubsan, db disk, old analyzer, {batch}/{total_batches}", runs_on=RunnerLabels.AMD_MEDIUM, - requires=[ArtifactNames.CH_AMD_ASAN_UBSAN], + requires=[ArtifactNames.CH_AMD_ASAN_UBSAN_GH], ) for total_batches in (6,) for batch in range(1, total_batches + 1) @@ -945,7 +977,7 @@ class JobConfigs: Job.ParamSet( parameter=f"arm_binary, distributed plan, {batch}/{total_batches}", runs_on=RunnerLabels.ARM_MEDIUM, - requires=[ArtifactNames.CH_ARM_BINARY], + requires=[ArtifactNames.CH_ARM_BINARY_GH], ) for total_batches in (4,) for batch in range(1, total_batches + 1) @@ -956,7 +988,7 @@ class JobConfigs: Job.ParamSet( parameter=f"amd_tsan, {batch}/{total_batches}", runs_on=RunnerLabels.AMD_MEDIUM, - requires=[ArtifactNames.CH_AMD_TSAN], + requires=[ArtifactNames.CH_AMD_TSAN_GH], ) for total_batches in (6,) for batch in range(1, total_batches + 1) @@ -965,7 +997,7 @@ class JobConfigs: Job.ParamSet( parameter=f"amd_msan, {batch}/{total_batches}", runs_on=RunnerLabels.AMD_MEDIUM, - requires=[ArtifactNames.CH_AMD_MSAN], + requires=[ArtifactNames.CH_AMD_MSAN_GH], ) for total_batches in (6,) for batch in range(1, total_batches + 1) @@ -976,7 +1008,7 @@ class JobConfigs: Job.ParamSet( parameter="amd_asan_ubsan, flaky", runs_on=RunnerLabels.AMD_MEDIUM, - requires=[ArtifactNames.CH_AMD_ASAN_UBSAN], + requires=[ArtifactNames.CH_AMD_ASAN_UBSAN_GH], ) ) ) @@ -1041,7 +1073,7 @@ class JobConfigs: Job.ParamSet( parameter="amd_binary_excluded_from_llvm", runs_on=RunnerLabels.AMD_MEDIUM, - requires=[ArtifactNames.CH_AMD_BINARY], + requires=[ArtifactNames.CH_AMD_BINARY_GH], ), ) @@ -1050,7 +1082,7 @@ class JobConfigs: Job.ParamSet( parameter="amd_binary_excluded_from_llvm", runs_on=RunnerLabels.AMD_MEDIUM, - requires=[ArtifactNames.CH_AMD_BINARY], + requires=[ArtifactNames.CH_AMD_BINARY_GH], ), ) ) @@ -1059,7 +1091,7 @@ class JobConfigs: Job.ParamSet( parameter="amd_asan_ubsan, targeted", runs_on=RunnerLabels.AMD_MEDIUM, - requires=[ArtifactNames.CH_AMD_ASAN_UBSAN], + requires=[ArtifactNames.CH_AMD_ASAN_UBSAN_GH], ) ) # Keeper stress job config — shared by PR and nightly workflows. @@ -1082,7 +1114,7 @@ class JobConfigs: "./src/Coordination/", ], ), - requires=[ArtifactNames.CH_ARM_BINARY], + requires=[ArtifactNames.CH_ARM_BINARY_GH], result_name_for_cidb="Keeper Stress", timeout=24 * 3600, post_hooks=["python3 ./ci/jobs/scripts/ingest_keeper_metrics.py"], @@ -1126,22 +1158,22 @@ class JobConfigs: Job.ParamSet( parameter="amd_debug", runs_on=RunnerLabels.FUNC_TESTER_AMD, - requires=[ArtifactNames.CH_AMD_DEBUG], + requires=[ArtifactNames.CH_AMD_DEBUG_GH], ), Job.ParamSet( parameter="arm_asan_ubsan", runs_on=RunnerLabels.FUNC_TESTER_ARM, - requires=[ArtifactNames.CH_ARM_ASAN_UBSAN], + requires=[ArtifactNames.CH_ARM_ASAN_UBSAN_GH], ), Job.ParamSet( parameter="amd_tsan", runs_on=RunnerLabels.FUNC_TESTER_AMD, - requires=[ArtifactNames.CH_AMD_TSAN], + requires=[ArtifactNames.CH_AMD_TSAN_GH], ), Job.ParamSet( parameter="amd_msan", runs_on=RunnerLabels.FUNC_TESTER_AMD, - requires=[ArtifactNames.CH_AMD_MSAN], + requires=[ArtifactNames.CH_AMD_MSAN_GH], ), ) ast_fuzzer_targeted_pr_jobs = Job.Config( @@ -1164,12 +1196,12 @@ class JobConfigs: Job.ParamSet( parameter="amd_debug, targeted", runs_on=RunnerLabels.FUNC_TESTER_AMD, - requires=[ArtifactNames.CH_AMD_DEBUG], + requires=[ArtifactNames.CH_AMD_DEBUG_GH], ), Job.ParamSet( parameter="amd_debug, targeted, old_compatibility", runs_on=RunnerLabels.FUNC_TESTER_AMD, - requires=[ArtifactNames.CH_AMD_DEBUG], + requires=[ArtifactNames.CH_AMD_DEBUG_GH], ), ) @@ -1192,22 +1224,22 @@ class JobConfigs: Job.ParamSet( parameter="amd_debug", runs_on=RunnerLabels.AMD_MEDIUM, - requires=[ArtifactNames.CH_AMD_DEBUG], + requires=[ArtifactNames.CH_AMD_DEBUG_GH], ), Job.ParamSet( parameter="arm_asan_ubsan", runs_on=RunnerLabels.ARM_MEDIUM, - requires=[ArtifactNames.CH_ARM_ASAN_UBSAN], + requires=[ArtifactNames.CH_ARM_ASAN_UBSAN_GH], ), Job.ParamSet( parameter="amd_tsan", runs_on=RunnerLabels.AMD_MEDIUM, - requires=[ArtifactNames.CH_AMD_TSAN], + requires=[ArtifactNames.CH_AMD_TSAN_GH], ), Job.ParamSet( parameter="amd_msan", runs_on=RunnerLabels.AMD_MEDIUM, - requires=[ArtifactNames.CH_AMD_MSAN], + requires=[ArtifactNames.CH_AMD_MSAN_GH], ), ) performance_comparison_with_master_head_jobs = Job.Config( @@ -1388,7 +1420,7 @@ class JobConfigs: Job.ParamSet( parameter="amd_debug", runs_on=RunnerLabels.FUNC_TESTER_AMD, - requires=[ArtifactNames.CH_AMD_DEBUG], + requires=[ArtifactNames.CH_AMD_DEBUG_GH], ), ) sqltest_master_job = Job.Config( diff --git a/ci/jobs/sqlstorm_test.py b/ci/jobs/sqlstorm_test.py index 1d25dece73bc..37520016e771 100644 --- a/ci/jobs/sqlstorm_test.py +++ b/ci/jobs/sqlstorm_test.py @@ -126,17 +126,6 @@ def create_log_export_config(self): # so the `Distributed` engine used by the log senders can resolve the # central CI logs cluster. Must run before the server is started. print("Create log export config") - config_file = Path(self.config_path) / "config.d" / "system_logs_export.yaml" - config_file.parent.mkdir(parents=True, exist_ok=True) - - # The log-replication materialized views are created with - # `DEFINER = ci_logs_sender`, so that user must exist on the server. - Shell.check( - f"mkdir -p {self.config_path}/users.d" - f" && cp ./tests/config/users.d/ci_logs_sender.yaml {self.config_path}/users.d/", - verbose=True, - strict=True, - ) self.log_export_host, self.log_export_password = ( Secret.Config( @@ -154,6 +143,18 @@ def create_log_export_config(self): .get_value() ) + config_file = Path(self.config_path) / "config.d" / "system_logs_export.yaml" + config_file.parent.mkdir(parents=True, exist_ok=True) + + # The log-replication materialized views are created with + # `DEFINER = ci_logs_sender`, so that user must exist on the server. + Shell.check( + f"mkdir -p {self.config_path}/users.d" + f" && cp ./tests/config/users.d/ci_logs_sender.yaml {self.config_path}/users.d/", + verbose=True, + strict=True, + ) + config_content = LOG_EXPORT_CONFIG_TEMPLATE.format( CLICKHOUSE_CI_LOGS_CLUSTER=CLICKHOUSE_CI_LOGS_CLUSTER, CLICKHOUSE_CI_LOGS_HOST=self.log_export_host, @@ -354,13 +355,21 @@ def main(): def start(): if not ch.install(): return False - # Configure export of system log tables to the central CI logs cluster - # (skipped for local runs, where the credentials are not available). + # Configure export of system log tables to the central CI logs cluster. + # Skip only when AWS credentials are absent in the environment. if not info.is_local_run: - ch.create_log_export_config() + aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID") + aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY") + if aws_access_key_id and aws_secret_access_key: + ch.create_log_export_config() + else: + print( + "WARNING: AWS credentials are not available, " + "skipping system log export setup" + ) if not ch.start(): return False - if not info.is_local_run: + if ch.log_export_host: if not ch.start_log_exports(check_start_time=stop_watch.start_time): print("WARNING: Failed to start log export") return True @@ -550,7 +559,8 @@ def report_and_thresholds(): ) # Detach log replication (flushes the remaining records first). - if not info.is_local_run: + # Only meaningful when log export was actually configured. + if ch.log_export_host: ch.stop_log_exports() Result.create_from( diff --git a/ci/praktika/runner.py b/ci/praktika/runner.py index 6a7d467849b4..8985dc03dbb8 100644 --- a/ci/praktika/runner.py +++ b/ci/praktika/runner.py @@ -204,16 +204,17 @@ def _pre_run(self, workflow, job, local_run=False): if job.requires and not _is_praktika_job(job.name): print("Download required artifacts") required_artifacts = [] + required_gh_artifacts = [] job_names_with_provides = { j.name for j in workflow.jobs if j.provides } for requires_artifact_name in job.requires: for artifact in workflow.artifacts: - if ( - artifact.name == requires_artifact_name - and artifact.type == Artifact.Type.S3 - ): - required_artifacts.append(artifact) + if artifact.name == requires_artifact_name: + if artifact.type == Artifact.Type.S3: + required_artifacts.append(artifact) + elif artifact.type == Artifact.Type.GH: + required_gh_artifacts.append(artifact) break else: if requires_artifact_name in job_names_with_provides: @@ -268,6 +269,88 @@ def _pre_run(self, workflow, job, local_run=False): if artifact.compress_zst: Utils.decompress_file(Path(Settings.INPUT_DIR) / artifact_path) + # GH artifacts are downloaded by workflow YAML steps. If they are missing (e.g. rerun after + # short GH retention expiry), attempt fallback to matching S3 artifact by name. + for gh_artifact in required_gh_artifacts: + if isinstance(gh_artifact.path, (tuple, list)): + gh_paths = gh_artifact.path + else: + gh_paths = [gh_artifact.path] + + expected_local_paths = [] + unresolved_pattern = False + for gh_path in gh_paths: + if "*" in gh_path: + unresolved_pattern = True + continue + expected_local_paths.append(Path(Settings.INPUT_DIR) / Path(gh_path).name) + + if expected_local_paths and all(p.exists() for p in expected_local_paths): + continue + + if unresolved_pattern and not expected_local_paths: + print( + f"WARNING: Cannot resolve expected local path for GH artifact [{gh_artifact.name}] with wildcard path [{gh_artifact.path}] - skip S3 fallback" + ) + continue + + s3_fallback_name = ( + gh_artifact.name[:-3] + if gh_artifact.name.endswith("_GH") + else gh_artifact.name + ) + s3_fallback_artifact = None + for artifact in workflow.artifacts: + if ( + artifact.name == s3_fallback_name + and artifact.type == Artifact.Type.S3 + ): + s3_fallback_artifact = artifact + break + + if not s3_fallback_artifact: + print( + f"WARNING: GH artifact [{gh_artifact.name}] is missing and no S3 fallback artifact [{s3_fallback_name}] is configured" + ) + continue + + print( + f"GH artifact [{gh_artifact.name}] is missing; fallback to S3 artifact [{s3_fallback_artifact.name}]" + ) + fallback_prefix = env.get_s3_prefix() + + fallback_artifact = dataclasses.replace(s3_fallback_artifact) + if fallback_artifact.compress_zst: + assert not isinstance( + fallback_artifact.path, (tuple, list) + ), "Not yes supported for compressed artifacts" + fallback_artifact.path = f"{Path(fallback_artifact.path).name}.zst" + + if isinstance(fallback_artifact.path, (tuple, list)): + fallback_paths = fallback_artifact.path + else: + fallback_paths = [fallback_artifact.path] + + for fallback_path in fallback_paths: + recursive = False + include_pattern = "" + if "*" in fallback_path: + s3_path = f"{Settings.S3_ARTIFACT_PATH}/{fallback_prefix}/{Utils.normalize_string(fallback_artifact._provided_by)}/" + recursive = True + include_pattern = Path(fallback_path).name + assert "*" in include_pattern + else: + s3_path = f"{Settings.S3_ARTIFACT_PATH}/{fallback_prefix}/{Utils.normalize_string(fallback_artifact._provided_by)}/{Path(fallback_path).name}" + S3.copy_file_from_s3( + s3_path=s3_path, + local_path=Settings.INPUT_DIR, + recursive=recursive, + include_pattern=include_pattern, + ) + + if fallback_artifact.compress_zst: + Utils.decompress_file(Path(Settings.INPUT_DIR) / fallback_path) + if not local_run and job.needs_submodules and Settings.ENABLE_SUBMODULE_CACHE: self._restore_submodule_cache() diff --git a/ci/praktika/validator.py b/ci/praktika/validator.py index ab7f41382f29..110427376d7e 100644 --- a/ci/praktika/validator.py +++ b/ci/praktika/validator.py @@ -212,8 +212,8 @@ def is_valid_cron_field(field: str) -> bool: if workflow.enable_cache: for artifact in workflow.artifacts or []: assert ( - artifact.is_s3_artifact() - ), f"All artifacts must be of S3 type if enable_cache|enable_html=True, artifact [{artifact.name}], type [{artifact.type}], workflow [{workflow.name}]" + artifact.is_s3_artifact() or artifact.type == Artifact.Type.GH + ), f"Artifacts must be S3 or GH type if enable_cache|enable_html=True, artifact [{artifact.name}], type [{artifact.type}], workflow [{workflow.name}]" if workflow.dockers and not workflow.disable_dockers_build: assert ( diff --git a/ci/praktika/yaml_additional_templates.py b/ci/praktika/yaml_additional_templates.py index ac694f442a16..c25c09f0fb38 100644 --- a/ci/praktika/yaml_additional_templates.py +++ b/ci/praktika/yaml_additional_templates.py @@ -75,7 +75,7 @@ class AltinityWorkflowTemplates: workflow_config: ${{ needs.config_workflow.outputs.data }} RegressionTestsAarch64: needs: [config_workflow, build_arm_binary, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.custom_data.ci_exclude_tags, 'regression') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.custom_data.ci_exclude_tags, 'aarch64')}} + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(fromJson(needs.config_workflow.outputs.data).JOB_KV_DATA.ci_exclude_tags, 'regression') && !contains(fromJson(needs.config_workflow.outputs.data).JOB_KV_DATA.ci_exclude_tags, 'aarch64')}} uses: ./.github/workflows/regression.yml secrets: inherit with: diff --git a/ci/praktika/yaml_generator.py b/ci/praktika/yaml_generator.py index 5e138efc8459..7d7fa7b86caa 100644 --- a/ci/praktika/yaml_generator.py +++ b/ci/praktika/yaml_generator.py @@ -221,7 +221,21 @@ class Templates: TEMPLATE_GH_UPLOAD = """ - name: Upload artifact {NAME} - uses: actions/upload-artifact@v4 + id: upload_{NAME} + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: {NAME} + path: {PATH} + retention-days: 1 + - name: Warn on failed upload of {NAME} + if: steps.upload_{NAME}.outcome == 'failure' + run: echo "::warning title=GH artifact upload failed::Failed to upload [{NAME}] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." +""" + + TEMPLATE_GH_UPLOAD_NO_RETENTION = """ + - name: Upload artifact {NAME} + uses: actions/upload-artifact@v7 with: name: {NAME} path: {PATH} @@ -229,7 +243,16 @@ class Templates: TEMPLATE_GH_DOWNLOAD = """ - name: Download artifact {NAME} - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: {NAME} + path: {PATH} +""" + + TEMPLATE_GH_DOWNLOAD_STRICT = """ + - name: Download artifact {NAME} + uses: actions/download-artifact@v8 with: name: {NAME} path: {PATH} @@ -323,16 +346,26 @@ def _all_needs(job_name: str) -> set: ) uploads_github = [] for artifact in job.artifacts_gh_provides: + upload_template = YamlGenerator.Templates.TEMPLATE_GH_UPLOAD + if self.workflow_config.name == "Community PR": + upload_template = ( + YamlGenerator.Templates.TEMPLATE_GH_UPLOAD_NO_RETENTION + ) uploads_github.append( - YamlGenerator.Templates.TEMPLATE_GH_UPLOAD.format( + upload_template.format( NAME=artifact.name, PATH=os.path.relpath(artifact.path, os.getcwd()), ) ) downloads_github = [] for artifact in job.artifacts_gh_requires: + download_template = YamlGenerator.Templates.TEMPLATE_GH_DOWNLOAD + if self.workflow_config.name == "Community PR": + download_template = ( + YamlGenerator.Templates.TEMPLATE_GH_DOWNLOAD_STRICT + ) downloads_github.append( - YamlGenerator.Templates.TEMPLATE_GH_DOWNLOAD.format( + download_template.format( NAME=artifact.name, PATH=Settings.INPUT_DIR ) ) @@ -345,7 +378,7 @@ def _all_needs(job_name: str) -> set: # NOTE (strtgbb): We still want the cache logic, we use it for skipping based on PR config if ( # self.workflow_config.config.enable_cache - # and + # and job_name_normalized != config_job_name_normalized ): if_expression = YamlGenerator.Templates.TEMPLATE_IF_EXPRESSION.format( diff --git a/ci/workflows/master.py b/ci/workflows/master.py index 6e414f2c17e7..0970e78cfc81 100644 --- a/ci/workflows/master.py +++ b/ci/workflows/master.py @@ -85,6 +85,7 @@ artifacts=[ *ArtifactConfigs.unittests_binaries, *clickhouse_binaries_with_tags, + *ArtifactConfigs.clickhouse_binaries_gh, *ArtifactConfigs.clickhouse_debians, *ArtifactConfigs.clickhouse_rpms, *ArtifactConfigs.clickhouse_tgzs, diff --git a/ci/workflows/nightly_keeper.py b/ci/workflows/nightly_keeper.py index 14089695aaa3..ce5910b7e583 100644 --- a/ci/workflows/nightly_keeper.py +++ b/ci/workflows/nightly_keeper.py @@ -5,7 +5,7 @@ binary_build_job = Job.Config.get_job( JobConfigs.build_jobs, f"Build ({BuildTypes.ARM_BINARY})" -).set_provides(ArtifactNames.CH_ARM_BINARY, reset=True) +).set_provides(ArtifactNames.CH_ARM_BINARY_GH, reset=True) # No-fault run: Sun / Tue / Thu / Sat workflow = Workflow.Config( @@ -18,6 +18,7 @@ ], artifacts=[ *ArtifactConfigs.clickhouse_binaries, + *ArtifactConfigs.clickhouse_binaries_gh, ], dockers=DOCKERS, secrets=SECRETS, diff --git a/ci/workflows/nightly_keeper_faults.py b/ci/workflows/nightly_keeper_faults.py index 38c15f0349a7..afc1b5d947b2 100644 --- a/ci/workflows/nightly_keeper_faults.py +++ b/ci/workflows/nightly_keeper_faults.py @@ -5,7 +5,7 @@ binary_build_job = Job.Config.get_job( JobConfigs.build_jobs, f"Build ({BuildTypes.ARM_BINARY})" -).set_provides(ArtifactNames.CH_ARM_BINARY, reset=True) +).set_provides(ArtifactNames.CH_ARM_BINARY_GH, reset=True) # Fault run: Mon / Wed / Fri workflow = Workflow.Config( @@ -18,6 +18,7 @@ ], artifacts=[ *ArtifactConfigs.clickhouse_binaries, + *ArtifactConfigs.clickhouse_binaries_gh, ], dockers=DOCKERS, secrets=SECRETS, diff --git a/ci/workflows/pull_request.py b/ci/workflows/pull_request.py index 008d02d6012f..45aa33bcb936 100644 --- a/ci/workflows/pull_request.py +++ b/ci/workflows/pull_request.py @@ -161,6 +161,7 @@ artifacts=[ *ArtifactConfigs.unittests_binaries, *ArtifactConfigs.clickhouse_binaries, + *ArtifactConfigs.clickhouse_binaries_gh, *ArtifactConfigs.clickhouse_stripped_binaries, *ArtifactConfigs.clickhouse_debians, *ArtifactConfigs.clickhouse_rpms, diff --git a/ci/workflows/pull_request_community.py b/ci/workflows/pull_request_community.py index 293916429c9a..af76e6ac9c4b 100644 --- a/ci/workflows/pull_request_community.py +++ b/ci/workflows/pull_request_community.py @@ -27,6 +27,19 @@ j for j in JobConfigs.functional_tests_jobs if "amd_debug, parallel" in j.name ][0] +def _normalize_gh_aliases(items): + if not items: + return items + normalized = [] + seen = set() + for item in items: + base = item[:-3] if item.endswith("_GH") else item + if base not in seen: + normalized.append(base) + seen.add(base) + return normalized + + workflow = Workflow.Config( name="Community PR", event=Workflow.Event.PULL_REQUEST, @@ -134,6 +147,8 @@ for i, job in enumerate(workflow.jobs): workflow.jobs[i] = copy.deepcopy(job) workflow.jobs[i].enable_commit_status = False + workflow.jobs[i].provides = _normalize_gh_aliases(workflow.jobs[i].provides) + workflow.jobs[i].requires = _normalize_gh_aliases(workflow.jobs[i].requires) for i, artifact in enumerate(workflow.artifacts): workflow.artifacts[i] = copy.deepcopy(artifact) diff --git a/ci/workflows/release_branches.py b/ci/workflows/release_branches.py index fb1150cd1750..2969c8f10b30 100644 --- a/ci/workflows/release_branches.py +++ b/ci/workflows/release_branches.py @@ -53,6 +53,7 @@ artifacts=[ *ArtifactConfigs.unittests_binaries, *clickhouse_binaries_with_tags, + *ArtifactConfigs.clickhouse_binaries_gh, *ArtifactConfigs.clickhouse_debians, *ArtifactConfigs.clickhouse_rpms, *ArtifactConfigs.clickhouse_tgzs, diff --git a/ci/workflows/release_builds.py b/ci/workflows/release_builds.py index bf4aee074cb5..5e6165f9a744 100644 --- a/ci/workflows/release_builds.py +++ b/ci/workflows/release_builds.py @@ -52,6 +52,7 @@ additional_jobs=["GrypeScan", "CIReport"], artifacts=[ *clickhouse_binaries_with_tags, + *ArtifactConfigs.clickhouse_binaries_gh, *ArtifactConfigs.clickhouse_debians, *ArtifactConfigs.clickhouse_rpms, *ArtifactConfigs.clickhouse_tgzs,