Skip to content

Commit 186efc1

Browse files
strtgbbcursoragent
andcommitted
Port SignRelease and SourceUpload additional jobs to praktika
The `SignRelease` (`reusable_sign.yml`) additional job was broken: it matrixed over a deprecated `jobs_data.jobs_params[...].batches` field that evaluated to an empty value (failing `MasterCI`'s strategy step) and called `tests/ci/ci.py`, which no longer exists. Rather than patch dead code, port both `SignRelease` and `SourceUpload` to native praktika jobs. - Add `ci/jobs/sign_release.py`: hashes release packages (deb/rpm/tgz and the self-extracting `clickhouse` binary) with SHA512 and GPG-signs the hash files, passing the passphrase via a file so it never hits the logs. - Add `ci/jobs/source_upload.py`: checks out submodules (praktika's job checkout has none, unlike the old `Altinity/checkout`), tars the tree and uploads it to the unchanged legacy S3 path so external consumers keep working. No `digest_config` so the CI cache never skips it across commits. - Add `ci/defs/altinity_jobs.py` to hold Altinity-only jobs and their artifacts (`sign_release_jobs`, `source_upload_job`, `signed_hashes`), keeping the upstream-tracked `defs.py`/`job_configs.py` merge-clean. - Wire the new jobs into `master`, `release_builds`, and (source upload only) `pull_request`; drop the ported entries from `additional_jobs`. - Remove the `SignRelease`/`SourceUpload` templates and generator wiring, the obsolete `reusable_sign.yml`, and the old `tests/ci/sign_release.py`. - Regenerate the workflow YAML. Ref: https://github.com/Altinity/ClickHouse/actions/runs/28142888526 Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 17af4ba commit 186efc1

14 files changed

Lines changed: 570 additions & 479 deletions

.github/workflows/master.yml

Lines changed: 141 additions & 57 deletions
Large diffs are not rendered by default.

.github/workflows/pull_request.yml

Lines changed: 49 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ env:
2626
CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }}
2727
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
2828
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
29+
GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }}
30+
GPG_BINARY_SIGNING_PASSPHRASE: ${{ secrets.GPG_BINARY_SIGNING_PASSPHRASE }}
2931
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
3032
CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }}
3133
AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }}
@@ -5442,9 +5444,54 @@ jobs:
54425444
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
54435445
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log
54445446
5447+
source_upload:
5448+
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
5449+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
5450+
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U291cmNlIHVwbG9hZA==') }}
5451+
name: "Source upload"
5452+
outputs:
5453+
data: ${{ steps.run.outputs.DATA }}
5454+
pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }}
5455+
steps:
5456+
- name: Checkout code
5457+
uses: actions/checkout@v6
5458+
with:
5459+
ref: ${{ env.CHECKOUT_REF }}
5460+
5461+
- name: Setup
5462+
uses: ./.github/actions/runner_setup
5463+
- name: Docker setup
5464+
uses: ./.github/actions/docker_setup
5465+
with:
5466+
test_name: "Source upload"
5467+
5468+
- name: Prepare env script
5469+
run: |
5470+
rm -rf ./ci/tmp
5471+
mkdir -p ./ci/tmp
5472+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
5473+
export PYTHONPATH=./ci:.:
5474+
5475+
cat > ./ci/tmp/workflow_job.json << 'EOF'
5476+
${{ toJson(job) }}
5477+
EOF
5478+
cat > ./ci/tmp/workflow_status.json << 'EOF'
5479+
${{ toJson(needs) }}
5480+
EOF
5481+
ENV_SETUP_SCRIPT_EOF
5482+
5483+
- name: Run
5484+
id: run
5485+
run: |
5486+
. ./ci/tmp/praktika_setup_env.sh
5487+
set -o pipefail
5488+
PYTHONUNBUFFERED=1 python3 -m praktika run 'Source upload' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime
5489+
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
5490+
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log
5491+
54455492
finish_workflow:
54465493
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
5447-
needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_ubsan, ast_fuzzer_arm_asan, build_amd_asan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_amd_ubsan, build_arm_asan, build_arm_binary, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_amd_ubsan, buzzhouse_arm_asan, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_db_disk_old_analyzer_1_6, integration_tests_amd_asan_db_disk_old_analyzer_2_6, integration_tests_amd_asan_db_disk_old_analyzer_3_6, integration_tests_amd_asan_db_disk_old_analyzer_4_6, integration_tests_amd_asan_db_disk_old_analyzer_5_6, integration_tests_amd_asan_db_disk_old_analyzer_6_6, integration_tests_amd_asan_targeted, integration_tests_amd_binary_1_5, integration_tests_amd_binary_2_5, integration_tests_amd_binary_3_5, integration_tests_amd_binary_4_5, integration_tests_amd_binary_5_5, integration_tests_amd_msan_1_6, integration_tests_amd_msan_2_6, integration_tests_amd_msan_3_6, integration_tests_amd_msan_4_6, integration_tests_amd_msan_5_6, integration_tests_amd_msan_6_6, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, quick_functional_tests, sqllogic_test, stateless_tests_amd_asan_db_disk_distributed_plan_sequential, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_amd_ubsan_parallel, stateless_tests_amd_ubsan_sequential, stateless_tests_arm_asan_azure_parallel_1_4, stateless_tests_arm_asan_azure_parallel_2_4, stateless_tests_arm_asan_azure_parallel_3_4, stateless_tests_arm_asan_azure_parallel_4_4, stateless_tests_arm_asan_azure_sequential_1_2, stateless_tests_arm_asan_azure_sequential_2_2, stateless_tests_arm_asan_targeted, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_release, stress_test_amd_tsan, stress_test_amd_ubsan, stress_test_arm_asan, stress_test_arm_asan_s3, unit_tests_asan, unit_tests_msan, unit_tests_tsan, unit_tests_ubsan]
5494+
needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_ubsan, ast_fuzzer_arm_asan, build_amd_asan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_amd_ubsan, build_arm_asan, build_arm_binary, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_amd_ubsan, buzzhouse_arm_asan, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_db_disk_old_analyzer_1_6, integration_tests_amd_asan_db_disk_old_analyzer_2_6, integration_tests_amd_asan_db_disk_old_analyzer_3_6, integration_tests_amd_asan_db_disk_old_analyzer_4_6, integration_tests_amd_asan_db_disk_old_analyzer_5_6, integration_tests_amd_asan_db_disk_old_analyzer_6_6, integration_tests_amd_asan_targeted, integration_tests_amd_binary_1_5, integration_tests_amd_binary_2_5, integration_tests_amd_binary_3_5, integration_tests_amd_binary_4_5, integration_tests_amd_binary_5_5, integration_tests_amd_msan_1_6, integration_tests_amd_msan_2_6, integration_tests_amd_msan_3_6, integration_tests_amd_msan_4_6, integration_tests_amd_msan_5_6, integration_tests_amd_msan_6_6, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, quick_functional_tests, source_upload, sqllogic_test, stateless_tests_amd_asan_db_disk_distributed_plan_sequential, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_amd_ubsan_parallel, stateless_tests_amd_ubsan_sequential, stateless_tests_arm_asan_azure_parallel_1_4, stateless_tests_arm_asan_azure_parallel_2_4, stateless_tests_arm_asan_azure_parallel_3_4, stateless_tests_arm_asan_azure_parallel_4_4, stateless_tests_arm_asan_azure_sequential_1_2, stateless_tests_arm_asan_azure_sequential_2_2, stateless_tests_arm_asan_targeted, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_release, stress_test_amd_tsan, stress_test_amd_ubsan, stress_test_arm_asan, stress_test_arm_asan_s3, unit_tests_asan, unit_tests_msan, unit_tests_tsan, unit_tests_ubsan]
54485495
if: ${{ always() && needs.config_workflow.outputs.pipeline_status != '' }}
54495496
name: "Finish Workflow"
54505497
outputs:
@@ -5652,6 +5699,7 @@ jobs:
56525699
- sqllogic_test
56535700
- build_toolchain_pgo_bolt_amd64
56545701
- build_toolchain_pgo_bolt_aarch64
5702+
- source_upload
56555703
- finish_workflow
56565704
- GrypeScanServer
56575705
- GrypeScanKeeper
@@ -5669,38 +5717,3 @@ jobs:
56695717
with:
56705718
workflow_config: ${{ toJson(needs) }}
56715719
final: true
5672-
5673-
SourceUpload:
5674-
needs: [config_workflow, build_amd_release]
5675-
if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' && !contains(needs.*.outputs.pipeline_status, 'failure') }}
5676-
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
5677-
env:
5678-
COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
5679-
PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
5680-
VERSION: ${{ fromJson(needs.config_workflow.outputs.data).JOB_KV_DATA.version.string }}
5681-
steps:
5682-
- name: Check out repository code
5683-
uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
5684-
with:
5685-
clear-repository: true
5686-
ref: ${{ fromJson(needs.config_workflow.outputs.data).git_ref }}
5687-
submodules: true
5688-
fetch-depth: 0
5689-
filter: tree:0
5690-
- name: Install aws cli
5691-
uses: unfor19/install-aws-cli-action@v1
5692-
with:
5693-
version: 2
5694-
arch: arm64
5695-
- name: Create source tar
5696-
run: |
5697-
cd .. && tar czf $RUNNER_TEMP/build_source.src.tar.gz ClickHouse/
5698-
- name: Upload source tar
5699-
run: |
5700-
if [ "$PR_NUMBER" -eq 0 ]; then
5701-
S3_PATH="REFs/$GITHUB_REF_NAME/$COMMIT_SHA/build_amd_release"
5702-
else
5703-
S3_PATH="PRs/$PR_NUMBER/$COMMIT_SHA/build_amd_release"
5704-
fi
5705-
5706-
aws s3 cp $RUNNER_TEMP/build_source.src.tar.gz s3://altinity-build-artifacts/$S3_PATH/clickhouse-$VERSION.src.tar.gz

0 commit comments

Comments
 (0)