Skip to content

Commit c8a7afc

Browse files
strtgbbstrtgbb
committed
fix workflows
1 parent e6b92da commit c8a7afc

9 files changed

Lines changed: 3515 additions & 4360 deletions

File tree

‎.github/workflows/master.yml‎

Lines changed: 1651 additions & 1876 deletions
Large diffs are not rendered by default.

‎.github/workflows/nightly_upload.yml‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,29 @@ concurrency:
1111

1212
env:
1313
PYTHONUNBUFFERED: 1
14+
GH_TOKEN: ${{ github.token }}
1415
CHECKOUT_REF: ""
16+
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
17+
CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}
18+
CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }}
19+
CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }}
20+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
21+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
22+
GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }}
23+
GPG_BINARY_SIGNING_PASSPHRASE: ${{ secrets.GPG_BINARY_SIGNING_PASSPHRASE }}
24+
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
25+
CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }}
26+
AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }}
27+
AZURE_ACCOUNT_NAME: ${{ secrets.AZURE_ACCOUNT_NAME }}
28+
AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }}
29+
AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/"
30+
ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
1531

1632

1733
jobs:
1834

1935
config_workflow:
20-
runs-on: [self-hosted, style-checker-aarch64]
36+
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
2137
needs: []
2238
name: "Config Workflow"
2339
outputs:
@@ -29,6 +45,27 @@ jobs:
2945
with:
3046
ref: ${{ env.CHECKOUT_REF }}
3147

48+
- name: Setup
49+
uses: ./.github/actions/runner_setup
50+
- name: Docker setup
51+
uses: ./.github/actions/docker_setup
52+
with:
53+
test_name: "Config Workflow"
54+
55+
- name: Note report location to summary
56+
if: ${{ !failure() && env.AWS_ACCESS_KEY_ID && env.AWS_SECRET_ACCESS_KEY }}
57+
env:
58+
PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
59+
COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
60+
run: |
61+
if [ "$PR_NUMBER" -eq 0 ]; then
62+
PREFIX="REFs/$GITHUB_REF_NAME/$COMMIT_SHA"
63+
else
64+
PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA"
65+
fi
66+
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html
67+
echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY
68+
3269
- name: Prepare env script
3370
run: |
3471
rm -rf ./ci/tmp
@@ -51,8 +88,9 @@ jobs:
5188
PYTHONUNBUFFERED=1 python3 -m praktika run 'Config Workflow' --workflow "NightlyUpload" --ci --timestamp
5289
5390
upload_clickhousectl:
54-
runs-on: [self-hosted, style-checker-aarch64]
91+
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
5592
needs: [config_workflow]
93+
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'VXBsb2FkIGNsaWNraG91c2VjdGw=') }}
5694
name: "Upload clickhousectl"
5795
outputs:
5896
data: ${{ steps.run.outputs.DATA }}
@@ -63,6 +101,13 @@ jobs:
63101
with:
64102
ref: ${{ env.CHECKOUT_REF }}
65103

104+
- name: Setup
105+
uses: ./.github/actions/runner_setup
106+
- name: Docker setup
107+
uses: ./.github/actions/docker_setup
108+
with:
109+
test_name: "Upload clickhousectl"
110+
66111
- name: Prepare env script
67112
run: |
68113
rm -rf ./ci/tmp

‎.github/workflows/optimize_clickhouse.yml‎

Lines changed: 80 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,30 @@ on:
77

88
env:
99
PYTHONUNBUFFERED: 1
10+
GH_TOKEN: ${{ github.token }}
1011
CHECKOUT_REF: ""
12+
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
13+
CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}
14+
CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }}
15+
CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }}
16+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
17+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
18+
GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }}
19+
GPG_BINARY_SIGNING_PASSPHRASE: ${{ secrets.GPG_BINARY_SIGNING_PASSPHRASE }}
20+
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
21+
CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }}
22+
AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }}
23+
AZURE_ACCOUNT_NAME: ${{ secrets.AZURE_ACCOUNT_NAME }}
24+
AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }}
25+
AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/"
26+
ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
27+
1128

12-
# Allow updating GH commit statuses and PR comments to post an actual job reports link
13-
permissions: write-all
1429

1530
jobs:
1631

1732
config_workflow:
18-
runs-on: [self-hosted, style-checker-aarch64]
33+
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
1934
needs: []
2035
name: "Config Workflow"
2136
outputs:
@@ -27,6 +42,27 @@ jobs:
2742
with:
2843
ref: ${{ env.CHECKOUT_REF }}
2944

45+
- name: Setup
46+
uses: ./.github/actions/runner_setup
47+
- name: Docker setup
48+
uses: ./.github/actions/docker_setup
49+
with:
50+
test_name: "Config Workflow"
51+
52+
- name: Note report location to summary
53+
if: ${{ !failure() && env.AWS_ACCESS_KEY_ID && env.AWS_SECRET_ACCESS_KEY }}
54+
env:
55+
PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
56+
COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
57+
run: |
58+
if [ "$PR_NUMBER" -eq 0 ]; then
59+
PREFIX="REFs/$GITHUB_REF_NAME/$COMMIT_SHA"
60+
else
61+
PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA"
62+
fi
63+
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html
64+
echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY
65+
3066
- name: Prepare env script
3167
run: |
3268
rm -rf ./ci/tmp
@@ -51,7 +87,7 @@ jobs:
5187
PYTHONUNBUFFERED=1 python3 -m praktika run 'Config Workflow' --workflow "OptimizeClickHouse" --ci --timestamp
5288
5389
dockers_build_amd:
54-
runs-on: [self-hosted, style-checker]
90+
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
5591
needs: [config_workflow]
5692
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'RG9ja2VycyBCdWlsZCAoYW1kKQ==') }}
5793
name: "Dockers Build (amd)"
@@ -64,6 +100,13 @@ jobs:
64100
with:
65101
ref: ${{ env.CHECKOUT_REF }}
66102

103+
- name: Setup
104+
uses: ./.github/actions/runner_setup
105+
- name: Docker setup
106+
uses: ./.github/actions/docker_setup
107+
with:
108+
test_name: "Dockers Build (amd)"
109+
67110
- name: Prepare env script
68111
run: |
69112
rm -rf ./ci/tmp
@@ -88,7 +131,7 @@ jobs:
88131
PYTHONUNBUFFERED=1 python3 -m praktika run 'Dockers Build (amd)' --workflow "OptimizeClickHouse" --ci --timestamp
89132
90133
dockers_build_arm:
91-
runs-on: [self-hosted, style-checker-aarch64]
134+
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
92135
needs: [config_workflow]
93136
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'RG9ja2VycyBCdWlsZCAoYXJtKQ==') }}
94137
name: "Dockers Build (arm)"
@@ -101,6 +144,13 @@ jobs:
101144
with:
102145
ref: ${{ env.CHECKOUT_REF }}
103146

147+
- name: Setup
148+
uses: ./.github/actions/runner_setup
149+
- name: Docker setup
150+
uses: ./.github/actions/docker_setup
151+
with:
152+
test_name: "Dockers Build (arm)"
153+
104154
- name: Prepare env script
105155
run: |
106156
rm -rf ./ci/tmp
@@ -125,7 +175,7 @@ jobs:
125175
PYTHONUNBUFFERED=1 python3 -m praktika run 'Dockers Build (arm)' --workflow "OptimizeClickHouse" --ci --timestamp
126176
127177
collect_clickhouse_profiles_pgo_bolt_amd64:
128-
runs-on: [self-hosted, amd-large]
178+
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester]
129179
needs: [config_workflow, dockers_build_amd, dockers_build_arm]
130180
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'Q29sbGVjdCBDbGlja0hvdXNlIFByb2ZpbGVzIChQR08sIEJPTFQpIChhbWQ2NCk=') }}
131181
name: "Collect ClickHouse Profiles (PGO, BOLT) (amd64)"
@@ -139,6 +189,13 @@ jobs:
139189
with:
140190
ref: ${{ env.CHECKOUT_REF }}
141191

192+
- name: Setup
193+
uses: ./.github/actions/runner_setup
194+
- name: Docker setup
195+
uses: ./.github/actions/docker_setup
196+
with:
197+
test_name: "Collect ClickHouse Profiles (PGO, BOLT) (amd64)"
198+
142199
- name: Prepare env script
143200
run: |
144201
rm -rf ./ci/tmp
@@ -163,7 +220,7 @@ jobs:
163220
PYTHONUNBUFFERED=1 python3 -m praktika run 'Collect ClickHouse Profiles (PGO, BOLT) (amd64)' --workflow "OptimizeClickHouse" --ci --timestamp
164221
165222
collect_clickhouse_profiles_pgo_bolt_aarch64:
166-
runs-on: [self-hosted, arm-large]
223+
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
167224
needs: [config_workflow, dockers_build_amd, dockers_build_arm]
168225
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'Q29sbGVjdCBDbGlja0hvdXNlIFByb2ZpbGVzIChQR08sIEJPTFQpIChhYXJjaDY0KQ==') }}
169226
name: "Collect ClickHouse Profiles (PGO, BOLT) (aarch64)"
@@ -177,6 +234,13 @@ jobs:
177234
with:
178235
ref: ${{ env.CHECKOUT_REF }}
179236

237+
- name: Setup
238+
uses: ./.github/actions/runner_setup
239+
- name: Docker setup
240+
uses: ./.github/actions/docker_setup
241+
with:
242+
test_name: "Collect ClickHouse Profiles (PGO, BOLT) (aarch64)"
243+
180244
- name: Prepare env script
181245
run: |
182246
rm -rf ./ci/tmp
@@ -201,9 +265,9 @@ jobs:
201265
PYTHONUNBUFFERED=1 python3 -m praktika run 'Collect ClickHouse Profiles (PGO, BOLT) (aarch64)' --workflow "OptimizeClickHouse" --ci --timestamp
202266
203267
finish_workflow:
204-
runs-on: [self-hosted, style-checker-aarch64]
268+
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
205269
needs: [collect_clickhouse_profiles_pgo_bolt_aarch64, collect_clickhouse_profiles_pgo_bolt_amd64, config_workflow, dockers_build_amd, dockers_build_arm]
206-
if: ${{ always() }}
270+
if: ${{ always() && needs.config_workflow.outputs.pipeline_status != '' }}
207271
name: "Finish Workflow"
208272
outputs:
209273
data: ${{ steps.run.outputs.DATA }}
@@ -214,6 +278,13 @@ jobs:
214278
with:
215279
ref: ${{ env.CHECKOUT_REF }}
216280

281+
- name: Setup
282+
uses: ./.github/actions/runner_setup
283+
- name: Docker setup
284+
uses: ./.github/actions/docker_setup
285+
with:
286+
test_name: "Finish Workflow"
287+
217288
- name: Prepare env script
218289
run: |
219290
rm -rf ./ci/tmp

‎.github/workflows/pr_version_info.yml‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,29 @@ concurrency:
1111

1212
env:
1313
PYTHONUNBUFFERED: 1
14+
GH_TOKEN: ${{ github.token }}
1415
CHECKOUT_REF: ""
16+
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
17+
CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}
18+
CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }}
19+
CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }}
20+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
21+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
22+
GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }}
23+
GPG_BINARY_SIGNING_PASSPHRASE: ${{ secrets.GPG_BINARY_SIGNING_PASSPHRASE }}
24+
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
25+
CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }}
26+
AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }}
27+
AZURE_ACCOUNT_NAME: ${{ secrets.AZURE_ACCOUNT_NAME }}
28+
AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }}
29+
AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/"
30+
ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
1531

1632

1733
jobs:
1834

1935
config_workflow:
20-
runs-on: [self-hosted, style-checker-aarch64]
36+
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
2137
needs: []
2238
name: "Config Workflow"
2339
outputs:
@@ -29,6 +45,27 @@ jobs:
2945
with:
3046
ref: ${{ env.CHECKOUT_REF }}
3147

48+
- name: Setup
49+
uses: ./.github/actions/runner_setup
50+
- name: Docker setup
51+
uses: ./.github/actions/docker_setup
52+
with:
53+
test_name: "Config Workflow"
54+
55+
- name: Note report location to summary
56+
if: ${{ !failure() && env.AWS_ACCESS_KEY_ID && env.AWS_SECRET_ACCESS_KEY }}
57+
env:
58+
PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
59+
COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
60+
run: |
61+
if [ "$PR_NUMBER" -eq 0 ]; then
62+
PREFIX="REFs/$GITHUB_REF_NAME/$COMMIT_SHA"
63+
else
64+
PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA"
65+
fi
66+
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html
67+
echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY
68+
3269
- name: Prepare env script
3370
run: |
3471
rm -rf ./ci/tmp
@@ -51,8 +88,9 @@ jobs:
5188
PYTHONUNBUFFERED=1 python3 -m praktika run 'Config Workflow' --workflow "PRVersionInfo" --ci --timestamp
5289
5390
update_pr_version_info:
54-
runs-on: [self-hosted, style-checker-aarch64]
91+
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
5592
needs: [config_workflow]
93+
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'VXBkYXRlIFBSIHZlcnNpb24gaW5mbw==') }}
5694
name: "Update PR version info"
5795
outputs:
5896
data: ${{ steps.run.outputs.DATA }}
@@ -63,6 +101,13 @@ jobs:
63101
with:
64102
ref: ${{ env.CHECKOUT_REF }}
65103

104+
- name: Setup
105+
uses: ./.github/actions/runner_setup
106+
- name: Docker setup
107+
uses: ./.github/actions/docker_setup
108+
with:
109+
test_name: "Update PR version info"
110+
66111
- name: Prepare env script
67112
run: |
68113
rm -rf ./ci/tmp

0 commit comments

Comments
 (0)