Skip to content

Commit 646d19c

Browse files
tommartensentommartensen
authored
fix: workflows use 'onExit: stop' handler to catch all errors during create (#1543)
1 parent 99cbf75 commit 646d19c

22 files changed

Lines changed: 189 additions & 14 deletions

‎.github/workflows/PR.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
with:
3535
flavor: gke-default
3636
name: infra-pr-${{ github.event.pull_request.number }}
37-
args: machine-type=e2-medium,nodes=3,gcp-image-type=ubuntu_containerd
37+
args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd
3838
lifespan: ${{ github.actor == 'dependabot[bot]' && '1h' || '24h' }}
3939
wait: true
4040
token: ${{ secrets.INFRA_TOKEN }}

‎.github/workflows/smoke-test.yaml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,9 @@ jobs:
4646
{name: "ocp-4", "args": "", "uniqueness": "g"},
4747
{name: "ocp-4-demo", "args": "", "uniqueness": "h"},
4848
{name: "ocp-4-perf-scale", "args": "", "uniqueness": "i"},
49-
{name: "ocp-3", "args": "", "uniqueness": "j"},
5049
{name: "osd-on-aws", "args": "", "uniqueness": "k"},
5150
{name: "osd-on-gcp", "args": "", "uniqueness": "l"},
52-
{name: "qa-demo", "args": "main-image=quay.io/rhacs-eng/main:4.3.4", "uniqueness": "m"},
51+
{name: "qa-demo", "args": "main-image=quay.io/rhacs-eng/main:4.7.0", "uniqueness": "m"},
5352
{name: "rosa", "args": "", "uniqueness": "n"},
5453
{name: "rosahcp", "args": "", "uniqueness": "o"},
5554
]

‎chart/infra-server/static/flavors.yaml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1243,20 +1243,24 @@
12431243
value: ""
12441244
- name: create-delay-seconds
12451245
description: how long to spend in create
1246-
value: "10"
1246+
value: "0"
12471247
kind: optional
12481248
- name: create-outcome
12491249
description: success or fail
12501250
value: success
12511251
kind: optional
12521252
- name: destroy-delay-seconds
12531253
description: how long to spend in destroy
1254-
value: "10"
1254+
value: "0"
12551255
kind: optional
12561256
- name: destroy-outcome
12571257
description: success or fail
12581258
value: success
12591259
kind: optional
1260+
- name: test-gcs
1261+
description: whether to upload or delete a test artifact in GCS
1262+
value: "false"
1263+
kind: optional
12601264

12611265
################
12621266
# Test Janitor #

‎chart/infra-server/static/test-connect-artifact.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ metadata:
44
generateName: test-connect-artifact-
55
spec:
66
entrypoint: start
7+
onExit: stop
78
arguments:
89
parameters:
910
- name: name
@@ -16,6 +17,8 @@ spec:
1617
template: create
1718
- - name: wait
1819
template: wait
20+
- name: stop
21+
steps:
1922
- - name: destroy
2023
template: destroy
2124

‎chart/infra-server/static/test-gke-lite.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ metadata:
44
generateName: gke-lite-
55
spec:
66
entrypoint: start
7+
onExit: stop
78
arguments:
89
parameters:
910
- name: name
@@ -30,12 +31,14 @@ spec:
3031
template: create
3132
- - name: wait
3233
template: wait
34+
- name: stop
35+
steps:
3336
- - name: destroy
3437
template: destroy
3538
arguments:
3639
parameters:
3740
- name: name
38-
value: '{{ "{{" }}steps.create.outputs.parameters.cluster_name{{ "}}" }}'
41+
value: '{{ "{{" }}workflow.parameters.name{{ "}}" }}'
3942

4043
- name: create
4144
activeDeadlineSeconds: 3600

‎chart/infra-server/static/test-qa-demo.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ metadata:
44
generateName: test-qa-demo-
55
spec:
66
entrypoint: start
7+
onExit: stop
78
arguments:
89
parameters:
910
- name: name
@@ -15,6 +16,8 @@ spec:
1516
template: whalesay
1617
- - name: wait
1718
template: wait
19+
- name: stop
20+
steps:
1821
- - name: destroy
1922
template: whalesay
2023

‎chart/infra-server/static/test-simulate.yaml‎

Lines changed: 56 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ metadata:
44
generateName: simulate-
55
spec:
66
entrypoint: start
7+
onExit: stop
78
arguments:
89
parameters:
910
- name: name
@@ -15,6 +16,13 @@ spec:
1516
value: ""
1617
- name: destroy-outcome
1718
value: ""
19+
- name: test-gcs
20+
value: ""
21+
22+
volumes:
23+
- name: credentials
24+
secret:
25+
secretName: google-credentials
1826

1927
templates:
2028
- name: start
@@ -27,8 +35,12 @@ spec:
2735
value: '{{ "{{" }}workflow.parameters.create-delay-seconds{{ "}}" }}'
2836
- name: outcome
2937
value: '{{ "{{" }}workflow.parameters.create-outcome{{ "}}" }}'
38+
- name: test-gcs
39+
value: '{{ "{{"}}workflow.parameters.test-gcs{{ "}}" }}'
3040
- - name: wait
3141
template: wait
42+
- name: stop
43+
steps:
3244
- - name: destroy
3345
template: simulate
3446
arguments:
@@ -37,24 +49,62 @@ spec:
3749
value: '{{ "{{" }}workflow.parameters.destroy-delay-seconds{{ "}}" }}'
3850
- name: outcome
3951
value: '{{ "{{" }}workflow.parameters.destroy-outcome{{ "}}" }}'
52+
- name: test-gcs
53+
value: '{{ "{{"}}workflow.parameters.test-gcs{{ "}}" }}'
4054

4155
- name: simulate
4256
inputs:
4357
parameters:
4458
- name: delay-seconds
4559
- name: outcome
60+
- name: test-gcs
4661
script:
47-
image: debian:9.4
62+
image: gcr.io/google.com/cloudsdktool/google-cloud-cli:stable
4863
command: [bash]
4964
source: |
5065
set -x
51-
start=0
52-
while sleep 1; do
53-
if [[ $((start++)) -ge {{ "{{" }}inputs.parameters.delay-seconds{{ "}}" }} ]]; then
54-
break
66+
67+
delay() {
68+
start=0
69+
while sleep 1; do
70+
if [[ $((start++)) -ge {{ "{{" }}inputs.parameters.delay-seconds{{ "}}" }} ]]; then
71+
break
72+
fi
73+
done
74+
}
75+
76+
upload_or_delete_gcs_object() {
77+
gcloud auth activate-service-account --key-file /tmp/google-credentials.json
78+
gcloud config set core/disable_prompts True
79+
80+
BUCKET_NAME="infra-e2e-upload-test"
81+
FILE="{{ "{{" }}workflow.name{{ "}}" }}"
82+
touch "${FILE}"
83+
84+
DESTINATION="gs://${BUCKET_NAME}/${FILE}"
85+
86+
if gsutil -q stat "${DESTINATION}"; then
87+
echo "File exists. Deleting..."
88+
gsutil rm "${DESTINATION}"
89+
else
90+
echo "File does not exist. Proceeding to upload."
91+
gsutil cp "${FILE}" "${DESTINATION}"
5592
fi
56-
done
93+
}
94+
95+
if [[ "{{ "{{" }}inputs.parameters.test-gcs{{ "}}" }}" == "true" ]]; then
96+
upload_or_delete_gcs_object
97+
fi
98+
99+
if [[ {{ "{{" }}inputs.parameters.delay-seconds{{ "}}" }} -gt 0 ]]; then
100+
delay
101+
fi
102+
57103
[[ "{{ "{{" }}inputs.parameters.outcome{{ "}}" }}" == "success" ]] || exit 1
58104
105+
volumeMounts:
106+
- name: credentials
107+
mountPath: /tmp
108+
59109
- name: wait
60110
suspend: {}

‎chart/infra-server/static/test-url-artifact.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ metadata:
44
generateName: test-url-artifact-
55
spec:
66
entrypoint: start
7+
onExit: stop
78
arguments:
89
parameters:
910
- name: name
@@ -16,6 +17,8 @@ spec:
1617
template: create
1718
- - name: wait
1819
template: wait
20+
- name: stop
21+
steps:
1922
- - name: destroy
2023
template: destroy
2124

‎chart/infra-server/static/workflow-aks.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ metadata:
44
generateName: aks-
55
spec:
66
entrypoint: start
7+
onExit: stop
78
arguments:
89
parameters:
910
- name: name
@@ -27,6 +28,8 @@ spec:
2728
- - name: wait
2829
template: wait
2930

31+
- name: stop
32+
steps:
3033
- - name: destroy
3134
template: destroy
3235

‎chart/infra-server/static/workflow-eks.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ metadata:
44
generateName: eks-
55
spec:
66
entrypoint: start
7+
onExit: stop
78
arguments:
89
parameters:
910
- name: name
@@ -27,6 +28,8 @@ spec:
2728
- - name: wait
2829
template: wait
2930

31+
- name: stop
32+
steps:
3033
- - name: destroy
3134
template: destroy
3235

0 commit comments

Comments
 (0)