From d2d1a6d786604b28822684ab600ac4294d5a79c9 Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Wed, 2 Apr 2025 13:14:36 +0200 Subject: [PATCH 1/9] spike: implement workflowtemplate --- .../static/workflow-openshift-4.yaml | 185 +++++------------- .../workflowtemplates/openshift-4.yaml | 153 +++++++++++++++ 2 files changed, 198 insertions(+), 140 deletions(-) create mode 100644 chart/infra-server/templates/workflowtemplates/openshift-4.yaml diff --git a/chart/infra-server/static/workflow-openshift-4.yaml b/chart/infra-server/static/workflow-openshift-4.yaml index 5f64bbdb2..c5222b236 100644 --- a/chart/infra-server/static/workflow-openshift-4.yaml +++ b/chart/infra-server/static/workflow-openshift-4.yaml @@ -8,19 +8,12 @@ spec: parameters: - name: name - name: openshift-version - value: "" - name: master-node-type - value: "" - name: master-node-count - value: "" - name: worker-node-type - value: "" - name: worker-node-count - value: "" - name: region - value: "" - name: pull-secret - value: "" - name: fips-enabled - name: trusted-certs-enabled - name: credentials-mode @@ -43,142 +36,54 @@ spec: - name: start steps: - - name: create - template: create + templateRef: + name: create + template: openshift-4 + arguments: + parameters: + - name: name + value: '{{ "{{" }}workflow.parameters.name{{ "}}" }}' + - name: pull-secret + value: '{{ "{{" }}workflow.parameters.pull-secret{{ "}}" }}' + - name: openshift-version + value: '{{ "{{" }}workflow.parameters.openshift-version{{ "}}" }}' + - name: master-node-count + value: '{{ "{{" }}workflow.parameters.master-node-count{{ "}}" }}' + - name: worker-node-count + value: '{{ "{{" }}workflow.parameters.worker-node-count{{ "}}" }}' + - name: master-node-type + value: '{{ "{{" }}workflow.parameters.master-node-type{{ "}}" }}' + - name: worker-node-type + value: '{{ "{{" }}workflow.parameters.worker-node-type{{ "}}" }}' + - name: region + value: '{{ "{{" }}workflow.parameters.region{{ "}}" }}' + - name: fips-enabled + value: '{{ "{{" }}workflow.parameters.fips-enabled{{ "}}" }}' + - name: trusted-certs-enabled + value: '{{ "{{" }}workflow.parameters.trusted-certs-enabled{{ "}}" }}' + - name: credentials-mode + value: '{{ "{{" }}workflow.parameters.credentials-mode{{ "}}" }}' + - name: keep-failed-cluster + value: '{{ "{{" }}workflow.parameters.keep-failed-cluster{{ "}}" }}' + - name: ssd-storage-class + value: '{{ "{{" }}workflow.parameters.ssd-storage-class{{ "}}" }}' + - - name: gather - template: gather + templateRef: + name: gather + template: openshift-4 - - name: wait - template: wait + templateRef: + name: wait + template: openshift-4 - - name: destroy - template: destroy - - - name: create - outputs: - artifacts: - - name: kubeconfig - path: /data/auth/kubeconfig - archive: - none: {} - - name: kubeadmin-password - path: /data/auth/kubeadmin-password - archive: - none: {} - - name: url - path: /data/url - archive: - none: {} - - name: dotenv - path: /data/dotenv - archive: - none: {} - - name: SSH_ACCESS - path: /data/ssh/SSH_ACCESS.md - archive: - none: {} - - name: data - path: /data - archive: - tar: {} - container: - image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} - imagePullPolicy: Always - command: - - entrypoint.sh - args: - - create - - '{{ "{{" }}workflow.parameters.name{{ "}}" }}' - - ocp.infra.rox.systems - env: - - name: GOOGLE_CREDENTIALS - valueFrom: - secretKeyRef: - name: openshift-4-gcp-service-account - key: google-credentials.json - - name: GCP_PROJECT - value : "acs-team-temp-dev" - - name: PULL_SECRET - valueFrom: - secretKeyRef: - name: redhat-pull-secret - key: REDHAT_PULL_SECRET - - name: USER_PULL_SECRET - value: '{{ "{{" }}workflow.parameters.pull-secret{{ "}}" }}' - - name: OPENSHIFT_VERSION - value: '{{ "{{" }}workflow.parameters.openshift-version{{ "}}" }}' - - name: MASTER_NODE_COUNT - value: '{{ "{{" }}workflow.parameters.master-node-count{{ "}}" }}' - - name: WORKER_NODE_COUNT - value: '{{ "{{" }}workflow.parameters.worker-node-count{{ "}}" }}' - - name: MASTER_NODE_TYPE - value: '{{ "{{" }}workflow.parameters.master-node-type{{ "}}" }}' - - name: WORKER_NODE_TYPE - value: '{{ "{{" }}workflow.parameters.worker-node-type{{ "}}" }}' - - name: REGION - value: '{{ "{{" }}workflow.parameters.region{{ "}}" }}' - - name: FIPS_ENABLED - value: '{{ "{{" }}workflow.parameters.fips-enabled{{ "}}" }}' - - name: TRUSTED_CERTS_ENABLED - value: '{{ "{{" }}workflow.parameters.trusted-certs-enabled{{ "}}" }}' - - name: CREDENTIALS_MODE - value: '{{ "{{" }}workflow.parameters.credentials-mode{{ "}}" }}' - - name: KEEP_FAILED_CLUSTER - value: '{{ "{{" }}workflow.parameters.keep-failed-cluster{{ "}}" }}' - - name: SSD_STORAGE_CLASS - value: '{{ "{{" }}workflow.parameters.ssd-storage-class{{ "}}" }}' - volumeMounts: - - name: data - mountPath: /data - - - name: gather - script: - image: busybox - command: [sh] - source: | - cd /data - . ./dotenv - echo "${OPENSHIFT_CONSOLE_URL}" > cluster-console-url - echo "${OPENSHIFT_CONSOLE_USERNAME}" > cluster-console-username - echo "${OPENSHIFT_CONSOLE_PASSWORD}" > cluster-console-password - volumeMounts: - - name: data - mountPath: /data - outputs: - artifacts: - - name: cluster-console-url - path: /data/cluster-console-url - archive: - none: {} - - name: cluster-console-username - path: /data/cluster-console-username - archive: - none: {} - - name: cluster-console-password - path: /data/cluster-console-password - archive: - none: {} - - - name: wait - suspend: {} - - - name: destroy - container: - image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} - imagePullPolicy: Always - command: - - entrypoint.sh - args: - - destroy - - '{{ "{{" }}workflow.parameters.name{{ "}}" }}' - env: - - name: GOOGLE_CREDENTIALS - valueFrom: - secretKeyRef: - name: openshift-4-gcp-service-account - key: google-credentials.json - - name: GCP_PROJECT - value : "acs-team-temp-dev" - volumeMounts: - - name: data - mountPath: /data + templateRef: + name: destroy + template: openshift-4 + arguments: + parameters: + - name: name + value: '{{ "{{" }}workflow.parameters.name{{ "}}" }}' diff --git a/chart/infra-server/templates/workflowtemplates/openshift-4.yaml b/chart/infra-server/templates/workflowtemplates/openshift-4.yaml new file mode 100644 index 000000000..71bd5d7a4 --- /dev/null +++ b/chart/infra-server/templates/workflowtemplates/openshift-4.yaml @@ -0,0 +1,153 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: openshift-4 +spec: + templates: + - name: create + inputs: + parameters: + - name: name + - name: openshift-version + - name: pull-secret + - name: master-node-count + - name: worker-node-count + - name: master-node-type + - name: worker-node-type + - name: region + - name: fips-enabled + - name: trusted-certs-enabled + - name: credentials-mode + - name: keep-failed-cluster + - name: ssd-storage-class + outputs: + artifacts: + - name: kubeconfig + path: /data/auth/kubeconfig + archive: + none: {} + - name: kubeadmin-password + path: /data/auth/kubeadmin-password + archive: + none: {} + - name: url + path: /data/url + archive: + none: {} + - name: dotenv + path: /data/dotenv + archive: + none: {} + - name: SSH_ACCESS + path: /data/ssh/SSH_ACCESS.md + archive: + none: {} + - name: data + path: /data + archive: + tar: {} + container: + image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} + imagePullPolicy: Always + command: + - entrypoint.sh + args: + - create + - '{{ "{{" }}inputs.parameters.name{{ "}}" }}' + - ocp.infra.rox.systems + env: + - name: GOOGLE_CREDENTIALS + valueFrom: + secretKeyRef: + name: openshift-4-gcp-service-account + key: google-credentials.json + - name: GCP_PROJECT + value : "acs-team-temp-dev" + - name: PULL_SECRET + valueFrom: + secretKeyRef: + name: redhat-pull-secret + key: REDHAT_PULL_SECRET + - name: USER_PULL_SECRET + value: '{{ "{{" }}inputs.parameters.pull-secret{{ "}}" }}' + - name: OPENSHIFT_VERSION + value: '{{ "{{" }}inputs.parameters.openshift-version{{ "}}" }}' + - name: MASTER_NODE_COUNT + value: '{{ "{{" }}inputs.parameters.master-node-count{{ "}}" }}' + - name: WORKER_NODE_COUNT + value: '{{ "{{" }}inputs.parameters.worker-node-count{{ "}}" }}' + - name: MASTER_NODE_TYPE + value: '{{ "{{" }}inputs.parameters.master-node-type{{ "}}" }}' + - name: WORKER_NODE_TYPE + value: '{{ "{{" }}inputs.parameters.worker-node-type{{ "}}" }}' + - name: REGION + value: '{{ "{{" }}inputs.parameters.region{{ "}}" }}' + - name: FIPS_ENABLED + value: '{{ "{{" }}inputs.parameters.fips-enabled{{ "}}" }}' + - name: TRUSTED_CERTS_ENABLED + value: '{{ "{{" }}inputs.parameters.trusted-certs-enabled{{ "}}" }}' + - name: CREDENTIALS_MODE + value: '{{ "{{" }}inputs.parameters.credentials-mode{{ "}}" }}' + - name: KEEP_FAILED_CLUSTER + value: '{{ "{{" }}inputs.parameters.keep-failed-cluster{{ "}}" }}' + - name: SSD_STORAGE_CLASS + value: '{{ "{{" }}inputs.parameters.ssd-storage-class{{ "}}" }}' + volumeMounts: + - name: data + mountPath: /data + + - name: gather + script: + image: busybox + command: [sh] + source: | + cd /data + . ./dotenv + echo "${OPENSHIFT_CONSOLE_URL}" > cluster-console-url + echo "${OPENSHIFT_CONSOLE_USERNAME}" > cluster-console-username + echo "${OPENSHIFT_CONSOLE_PASSWORD}" > cluster-console-password + volumeMounts: + - name: data + mountPath: /data + outputs: + artifacts: + - name: cluster-console-url + path: /data/cluster-console-url + archive: + none: {} + - name: cluster-console-username + path: /data/cluster-console-username + archive: + none: {} + - name: cluster-console-password + path: /data/cluster-console-password + archive: + none: {} + + - name: wait + suspend: {} + + - name: destroy + inputs: + parameters: + - name: name + container: + image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} + imagePullPolicy: Always + command: + - entrypoint.sh + args: + - destroy + - '{{ "{{" }}inputs.parameters.name{{ "}}" }}' + env: + - name: GOOGLE_CREDENTIALS + valueFrom: + secretKeyRef: + name: openshift-4-gcp-service-account + key: google-credentials.json + - name: GCP_PROJECT + value : "acs-team-temp-dev" + volumeMounts: + - name: data + mountPath: /data From 5f96a41257fd1651138147a2a51a8a48a5ca00b9 Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Wed, 2 Apr 2025 15:22:07 +0200 Subject: [PATCH 2/9] do example with simulate instead --- chart/infra-server/argo-values.yaml | 3 + chart/infra-server/static/test-simulate.yaml | 39 ++-- .../static/workflow-openshift-4.yaml | 185 +++++++++++++----- .../templates/workflowtemplates/common.yaml | 10 + .../workflowtemplates/openshift-4.yaml | 153 --------------- .../workflowtemplates/test-simulate.yaml | 25 +++ scripts/deploy/helm.sh | 7 + 7 files changed, 196 insertions(+), 226 deletions(-) create mode 100644 chart/infra-server/templates/workflowtemplates/common.yaml delete mode 100644 chart/infra-server/templates/workflowtemplates/openshift-4.yaml create mode 100644 chart/infra-server/templates/workflowtemplates/test-simulate.yaml diff --git a/chart/infra-server/argo-values.yaml b/chart/infra-server/argo-values.yaml index 0ac701bb8..c01ea380e 100644 --- a/chart/infra-server/argo-values.yaml +++ b/chart/infra-server/argo-values.yaml @@ -3,6 +3,9 @@ argo-workflows: server: authModes: - server + # We do this separately, because they cannot be installed from the dependant chart. + crds: + install: false controller: # Default values that will apply to all Workflows from this controller, unless overridden on the Workflow-level diff --git a/chart/infra-server/static/test-simulate.yaml b/chart/infra-server/static/test-simulate.yaml index 2e7f7cd41..8f633adba 100644 --- a/chart/infra-server/static/test-simulate.yaml +++ b/chart/infra-server/static/test-simulate.yaml @@ -8,53 +8,36 @@ spec: parameters: - name: name - name: create-delay-seconds - value: "" - name: create-outcome - value: "" - name: destroy-delay-seconds - value: "" - name: destroy-outcome - value: "" templates: - name: start steps: - - name: create - template: simulate + templateRef: + name: test-simulate + template: simulate arguments: parameters: - name: delay-seconds value: '{{ "{{" }}workflow.parameters.create-delay-seconds{{ "}}" }}' - name: outcome value: '{{ "{{" }}workflow.parameters.create-outcome{{ "}}" }}' + - - name: wait - template: wait + templateRef: + name: common + template: wait + - - name: destroy - template: simulate + templateRef: + name: test-simulate + template: simulate arguments: parameters: - name: delay-seconds value: '{{ "{{" }}workflow.parameters.destroy-delay-seconds{{ "}}" }}' - name: outcome value: '{{ "{{" }}workflow.parameters.destroy-outcome{{ "}}" }}' - - - name: simulate - inputs: - parameters: - - name: delay-seconds - - name: outcome - script: - image: debian:9.4 - command: [bash] - source: | - set -x - start=0 - while sleep 1; do - if [[ $((start++)) -ge {{ "{{" }}inputs.parameters.delay-seconds{{ "}}" }} ]]; then - break - fi - done - [[ "{{ "{{" }}inputs.parameters.outcome{{ "}}" }}" == "success" ]] || exit 1 - - - name: wait - suspend: {} diff --git a/chart/infra-server/static/workflow-openshift-4.yaml b/chart/infra-server/static/workflow-openshift-4.yaml index c5222b236..5f64bbdb2 100644 --- a/chart/infra-server/static/workflow-openshift-4.yaml +++ b/chart/infra-server/static/workflow-openshift-4.yaml @@ -8,12 +8,19 @@ spec: parameters: - name: name - name: openshift-version + value: "" - name: master-node-type + value: "" - name: master-node-count + value: "" - name: worker-node-type + value: "" - name: worker-node-count + value: "" - name: region + value: "" - name: pull-secret + value: "" - name: fips-enabled - name: trusted-certs-enabled - name: credentials-mode @@ -36,54 +43,142 @@ spec: - name: start steps: - - name: create - templateRef: - name: create - template: openshift-4 - arguments: - parameters: - - name: name - value: '{{ "{{" }}workflow.parameters.name{{ "}}" }}' - - name: pull-secret - value: '{{ "{{" }}workflow.parameters.pull-secret{{ "}}" }}' - - name: openshift-version - value: '{{ "{{" }}workflow.parameters.openshift-version{{ "}}" }}' - - name: master-node-count - value: '{{ "{{" }}workflow.parameters.master-node-count{{ "}}" }}' - - name: worker-node-count - value: '{{ "{{" }}workflow.parameters.worker-node-count{{ "}}" }}' - - name: master-node-type - value: '{{ "{{" }}workflow.parameters.master-node-type{{ "}}" }}' - - name: worker-node-type - value: '{{ "{{" }}workflow.parameters.worker-node-type{{ "}}" }}' - - name: region - value: '{{ "{{" }}workflow.parameters.region{{ "}}" }}' - - name: fips-enabled - value: '{{ "{{" }}workflow.parameters.fips-enabled{{ "}}" }}' - - name: trusted-certs-enabled - value: '{{ "{{" }}workflow.parameters.trusted-certs-enabled{{ "}}" }}' - - name: credentials-mode - value: '{{ "{{" }}workflow.parameters.credentials-mode{{ "}}" }}' - - name: keep-failed-cluster - value: '{{ "{{" }}workflow.parameters.keep-failed-cluster{{ "}}" }}' - - name: ssd-storage-class - value: '{{ "{{" }}workflow.parameters.ssd-storage-class{{ "}}" }}' - + template: create - - name: gather - templateRef: - name: gather - template: openshift-4 + template: gather - - name: wait - templateRef: - name: wait - template: openshift-4 + template: wait - - name: destroy - templateRef: - name: destroy - template: openshift-4 - arguments: - parameters: - - name: name - value: '{{ "{{" }}workflow.parameters.name{{ "}}" }}' + template: destroy + + - name: create + outputs: + artifacts: + - name: kubeconfig + path: /data/auth/kubeconfig + archive: + none: {} + - name: kubeadmin-password + path: /data/auth/kubeadmin-password + archive: + none: {} + - name: url + path: /data/url + archive: + none: {} + - name: dotenv + path: /data/dotenv + archive: + none: {} + - name: SSH_ACCESS + path: /data/ssh/SSH_ACCESS.md + archive: + none: {} + - name: data + path: /data + archive: + tar: {} + container: + image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} + imagePullPolicy: Always + command: + - entrypoint.sh + args: + - create + - '{{ "{{" }}workflow.parameters.name{{ "}}" }}' + - ocp.infra.rox.systems + env: + - name: GOOGLE_CREDENTIALS + valueFrom: + secretKeyRef: + name: openshift-4-gcp-service-account + key: google-credentials.json + - name: GCP_PROJECT + value : "acs-team-temp-dev" + - name: PULL_SECRET + valueFrom: + secretKeyRef: + name: redhat-pull-secret + key: REDHAT_PULL_SECRET + - name: USER_PULL_SECRET + value: '{{ "{{" }}workflow.parameters.pull-secret{{ "}}" }}' + - name: OPENSHIFT_VERSION + value: '{{ "{{" }}workflow.parameters.openshift-version{{ "}}" }}' + - name: MASTER_NODE_COUNT + value: '{{ "{{" }}workflow.parameters.master-node-count{{ "}}" }}' + - name: WORKER_NODE_COUNT + value: '{{ "{{" }}workflow.parameters.worker-node-count{{ "}}" }}' + - name: MASTER_NODE_TYPE + value: '{{ "{{" }}workflow.parameters.master-node-type{{ "}}" }}' + - name: WORKER_NODE_TYPE + value: '{{ "{{" }}workflow.parameters.worker-node-type{{ "}}" }}' + - name: REGION + value: '{{ "{{" }}workflow.parameters.region{{ "}}" }}' + - name: FIPS_ENABLED + value: '{{ "{{" }}workflow.parameters.fips-enabled{{ "}}" }}' + - name: TRUSTED_CERTS_ENABLED + value: '{{ "{{" }}workflow.parameters.trusted-certs-enabled{{ "}}" }}' + - name: CREDENTIALS_MODE + value: '{{ "{{" }}workflow.parameters.credentials-mode{{ "}}" }}' + - name: KEEP_FAILED_CLUSTER + value: '{{ "{{" }}workflow.parameters.keep-failed-cluster{{ "}}" }}' + - name: SSD_STORAGE_CLASS + value: '{{ "{{" }}workflow.parameters.ssd-storage-class{{ "}}" }}' + volumeMounts: + - name: data + mountPath: /data + + - name: gather + script: + image: busybox + command: [sh] + source: | + cd /data + . ./dotenv + echo "${OPENSHIFT_CONSOLE_URL}" > cluster-console-url + echo "${OPENSHIFT_CONSOLE_USERNAME}" > cluster-console-username + echo "${OPENSHIFT_CONSOLE_PASSWORD}" > cluster-console-password + volumeMounts: + - name: data + mountPath: /data + outputs: + artifacts: + - name: cluster-console-url + path: /data/cluster-console-url + archive: + none: {} + - name: cluster-console-username + path: /data/cluster-console-username + archive: + none: {} + - name: cluster-console-password + path: /data/cluster-console-password + archive: + none: {} + + - name: wait + suspend: {} + + - name: destroy + container: + image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} + imagePullPolicy: Always + command: + - entrypoint.sh + args: + - destroy + - '{{ "{{" }}workflow.parameters.name{{ "}}" }}' + env: + - name: GOOGLE_CREDENTIALS + valueFrom: + secretKeyRef: + name: openshift-4-gcp-service-account + key: google-credentials.json + - name: GCP_PROJECT + value : "acs-team-temp-dev" + volumeMounts: + - name: data + mountPath: /data diff --git a/chart/infra-server/templates/workflowtemplates/common.yaml b/chart/infra-server/templates/workflowtemplates/common.yaml new file mode 100644 index 000000000..a8f6bad03 --- /dev/null +++ b/chart/infra-server/templates/workflowtemplates/common.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: common + namespace: default +spec: + templates: + - name: wait + suspend: {} diff --git a/chart/infra-server/templates/workflowtemplates/openshift-4.yaml b/chart/infra-server/templates/workflowtemplates/openshift-4.yaml deleted file mode 100644 index 71bd5d7a4..000000000 --- a/chart/infra-server/templates/workflowtemplates/openshift-4.yaml +++ /dev/null @@ -1,153 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: openshift-4 -spec: - templates: - - name: create - inputs: - parameters: - - name: name - - name: openshift-version - - name: pull-secret - - name: master-node-count - - name: worker-node-count - - name: master-node-type - - name: worker-node-type - - name: region - - name: fips-enabled - - name: trusted-certs-enabled - - name: credentials-mode - - name: keep-failed-cluster - - name: ssd-storage-class - outputs: - artifacts: - - name: kubeconfig - path: /data/auth/kubeconfig - archive: - none: {} - - name: kubeadmin-password - path: /data/auth/kubeadmin-password - archive: - none: {} - - name: url - path: /data/url - archive: - none: {} - - name: dotenv - path: /data/dotenv - archive: - none: {} - - name: SSH_ACCESS - path: /data/ssh/SSH_ACCESS.md - archive: - none: {} - - name: data - path: /data - archive: - tar: {} - container: - image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} - imagePullPolicy: Always - command: - - entrypoint.sh - args: - - create - - '{{ "{{" }}inputs.parameters.name{{ "}}" }}' - - ocp.infra.rox.systems - env: - - name: GOOGLE_CREDENTIALS - valueFrom: - secretKeyRef: - name: openshift-4-gcp-service-account - key: google-credentials.json - - name: GCP_PROJECT - value : "acs-team-temp-dev" - - name: PULL_SECRET - valueFrom: - secretKeyRef: - name: redhat-pull-secret - key: REDHAT_PULL_SECRET - - name: USER_PULL_SECRET - value: '{{ "{{" }}inputs.parameters.pull-secret{{ "}}" }}' - - name: OPENSHIFT_VERSION - value: '{{ "{{" }}inputs.parameters.openshift-version{{ "}}" }}' - - name: MASTER_NODE_COUNT - value: '{{ "{{" }}inputs.parameters.master-node-count{{ "}}" }}' - - name: WORKER_NODE_COUNT - value: '{{ "{{" }}inputs.parameters.worker-node-count{{ "}}" }}' - - name: MASTER_NODE_TYPE - value: '{{ "{{" }}inputs.parameters.master-node-type{{ "}}" }}' - - name: WORKER_NODE_TYPE - value: '{{ "{{" }}inputs.parameters.worker-node-type{{ "}}" }}' - - name: REGION - value: '{{ "{{" }}inputs.parameters.region{{ "}}" }}' - - name: FIPS_ENABLED - value: '{{ "{{" }}inputs.parameters.fips-enabled{{ "}}" }}' - - name: TRUSTED_CERTS_ENABLED - value: '{{ "{{" }}inputs.parameters.trusted-certs-enabled{{ "}}" }}' - - name: CREDENTIALS_MODE - value: '{{ "{{" }}inputs.parameters.credentials-mode{{ "}}" }}' - - name: KEEP_FAILED_CLUSTER - value: '{{ "{{" }}inputs.parameters.keep-failed-cluster{{ "}}" }}' - - name: SSD_STORAGE_CLASS - value: '{{ "{{" }}inputs.parameters.ssd-storage-class{{ "}}" }}' - volumeMounts: - - name: data - mountPath: /data - - - name: gather - script: - image: busybox - command: [sh] - source: | - cd /data - . ./dotenv - echo "${OPENSHIFT_CONSOLE_URL}" > cluster-console-url - echo "${OPENSHIFT_CONSOLE_USERNAME}" > cluster-console-username - echo "${OPENSHIFT_CONSOLE_PASSWORD}" > cluster-console-password - volumeMounts: - - name: data - mountPath: /data - outputs: - artifacts: - - name: cluster-console-url - path: /data/cluster-console-url - archive: - none: {} - - name: cluster-console-username - path: /data/cluster-console-username - archive: - none: {} - - name: cluster-console-password - path: /data/cluster-console-password - archive: - none: {} - - - name: wait - suspend: {} - - - name: destroy - inputs: - parameters: - - name: name - container: - image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} - imagePullPolicy: Always - command: - - entrypoint.sh - args: - - destroy - - '{{ "{{" }}inputs.parameters.name{{ "}}" }}' - env: - - name: GOOGLE_CREDENTIALS - valueFrom: - secretKeyRef: - name: openshift-4-gcp-service-account - key: google-credentials.json - - name: GCP_PROJECT - value : "acs-team-temp-dev" - volumeMounts: - - name: data - mountPath: /data diff --git a/chart/infra-server/templates/workflowtemplates/test-simulate.yaml b/chart/infra-server/templates/workflowtemplates/test-simulate.yaml new file mode 100644 index 000000000..066a80d15 --- /dev/null +++ b/chart/infra-server/templates/workflowtemplates/test-simulate.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: test-simulate + namespace: default +spec: + templates: + - name: simulate + inputs: + parameters: + - name: delay-seconds + - name: outcome + script: + image: debian:9.4 + command: [bash] + source: | + set -x + start=0 + while sleep 1; do + if [[ $((start++)) -ge {{ "{{" }}inputs.parameters.delay-seconds{{ "}}" }} ]]; then + break + fi + done + [[ "{{ "{{" }}inputs.parameters.outcome{{ "}}" }}" == "success" ]] || exit 1 diff --git a/scripts/deploy/helm.sh b/scripts/deploy/helm.sh index df4ee2853..073f2d0cb 100755 --- a/scripts/deploy/helm.sh +++ b/scripts/deploy/helm.sh @@ -14,6 +14,7 @@ TEST_MODE="${TEST_MODE:-false}" SECRETS_PROJECT="acs-team-automation" RELEASE_NAMESPACE="infra" RELEASE_NAME="infra-server" +ARGO_WORKFLOWS_APP_VERSION="v3.6.5" check_not_empty() { for V in "$@"; do @@ -25,6 +26,11 @@ check_not_empty() { done } +install_crds() { + kubectl apply --kustomize \ + "https://github.com/argoproj/argo-workflows/manifests/base/crds/minimal?ref=${ARGO_WORKFLOWS_APP_VERSION}" +} + template() { # Need to use helm upgrade --dry-run to have .Capabilities context available helm upgrade \ @@ -102,4 +108,5 @@ diff() { } check_not_empty TASK TAG ENVIRONMENT +install_crds eval "$TASK" From c202f318fa81b67ede933e52c1b18180ba6a0577 Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Mon, 7 Apr 2025 13:25:26 +0200 Subject: [PATCH 3/9] dag and remove superfluous workflow --- chart/infra-server/static/flavors.yaml | 9 +- chart/infra-server/static/test-simulate.yaml | 14 +- .../workflow-openshift-4-perf-scale.yaml | 181 ------------------ chart/infra-server/templates/secrets.yaml | 3 - 4 files changed, 18 insertions(+), 189 deletions(-) delete mode 100644 chart/infra-server/static/workflow-openshift-4-perf-scale.yaml diff --git a/chart/infra-server/static/flavors.yaml b/chart/infra-server/static/flavors.yaml index cddd7572e..be73eb721 100644 --- a/chart/infra-server/static/flavors.yaml +++ b/chart/infra-server/static/flavors.yaml @@ -494,7 +494,7 @@ name: OpenShift 4.x Perf&Scale description: OpenShift 4.x Perf&Scale availability: stable - workflow: configuration/workflow-openshift-4-perf-scale.yaml + workflow: configuration/workflow-openshift-4.yaml aliases: - ocp-4-perf-scale parameters: @@ -579,6 +579,13 @@ Consult OCP documentation for details. {{ .Chart.Annotations.ocpCredentialsMode }} is the value used by stackrox CI. + - name: keep-failed-cluster + description: Keep failed cluster + value: false + kind: optional + help: | + *Only for debugging infra issues.* If in doubt, please keep to false. + - name: ssd-storage-class description: Ensure an SSD StorageClass is the default StorageClass for the cluster value: false diff --git a/chart/infra-server/static/test-simulate.yaml b/chart/infra-server/static/test-simulate.yaml index 8f633adba..61a77cfc7 100644 --- a/chart/infra-server/static/test-simulate.yaml +++ b/chart/infra-server/static/test-simulate.yaml @@ -4,6 +4,7 @@ metadata: generateName: simulate- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -14,8 +15,9 @@ spec: templates: - name: start - steps: - - - name: create + dag: + tasks: + - name: create templateRef: name: test-simulate template: simulate @@ -26,12 +28,16 @@ spec: - name: outcome value: '{{ "{{" }}workflow.parameters.create-outcome{{ "}}" }}' - - - name: wait + - name: wait + dependencies: [create] templateRef: name: common template: wait - - - name: destroy + - name: stop + dag: + tasks: + - name: destroy templateRef: name: test-simulate template: simulate diff --git a/chart/infra-server/static/workflow-openshift-4-perf-scale.yaml b/chart/infra-server/static/workflow-openshift-4-perf-scale.yaml deleted file mode 100644 index 3a0bdaaa6..000000000 --- a/chart/infra-server/static/workflow-openshift-4-perf-scale.yaml +++ /dev/null @@ -1,181 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: openshift-4-perf-scale- -spec: - entrypoint: start - arguments: - parameters: - - name: name - - name: openshift-version - value: "" - - name: master-node-type - value: "" - - name: master-node-count - value: "" - - name: worker-node-type - value: "" - - name: worker-node-count - value: "" - - name: region - value: "" - - name: pull-secret - value: "" - - name: fips-enabled - - name: trusted-certs-enabled - - name: credentials-mode - - name: ssd-storage-class - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 10Mi - volumes: - - name: credentials - secret: - secretName: openshift-4-gcp-service-account - - templates: - - name: start - steps: - - - name: create - template: create - - - - name: gather - template: gather - - - - name: wait - template: wait - - - - name: destroy - template: destroy - - - name: create - outputs: - artifacts: - - name: kubeconfig - path: /data/auth/kubeconfig - archive: - none: {} - - name: kubeadmin-password - path: /data/auth/kubeadmin-password - archive: - none: {} - - name: url - path: /data/url - archive: - none: {} - - name: dotenv - path: /data/dotenv - archive: - none: {} - - name: SSH_ACCESS - path: /data/ssh/SSH_ACCESS.md - archive: - none: {} - - name: data - path: /data - archive: - tar: {} - container: - image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} - imagePullPolicy: Always - command: - - entrypoint.sh - args: - - create - - '{{ "{{" }}workflow.parameters.name{{ "}}" }}' - - ocp.infra.rox.systems - env: - - name: GOOGLE_CREDENTIALS - valueFrom: - secretKeyRef: - name: openshift-4-gcp-service-account - key: google-credentials.json - - name: GCP_PROJECT - value : "acs-team-temp-dev" - - name: PULL_SECRET - valueFrom: - secretKeyRef: - name: redhat-pull-secret - key: REDHAT_PULL_SECRET - - name: USER_PULL_SECRET - value: '{{ "{{" }}workflow.parameters.pull-secret{{ "}}" }}' - - name: OPENSHIFT_VERSION - value: '{{ "{{" }}workflow.parameters.openshift-version{{ "}}" }}' - - name: MASTER_NODE_COUNT - value: '{{ "{{" }}workflow.parameters.master-node-count{{ "}}" }}' - - name: WORKER_NODE_COUNT - value: '{{ "{{" }}workflow.parameters.worker-node-count{{ "}}" }}' - - name: MASTER_NODE_TYPE - value: '{{ "{{" }}workflow.parameters.master-node-type{{ "}}" }}' - - name: WORKER_NODE_TYPE - value: '{{ "{{" }}workflow.parameters.worker-node-type{{ "}}" }}' - - name: REGION - value: '{{ "{{" }}workflow.parameters.region{{ "}}" }}' - - name: FIPS_ENABLED - value: '{{ "{{" }}workflow.parameters.fips-enabled{{ "}}" }}' - - name: TRUSTED_CERTS_ENABLED - value: '{{ "{{" }}workflow.parameters.trusted-certs-enabled{{ "}}" }}' - - name: CREDENTIALS_MODE - value: '{{ "{{" }}workflow.parameters.credentials-mode{{ "}}" }}' - - name: SSD_STORAGE_CLASS - value: '{{ "{{" }}workflow.parameters.ssd-storage-class{{ "}}" }}' - volumeMounts: - - name: data - mountPath: /data - - - name: gather - script: - image: busybox - command: [sh] - source: | - cd /data - . ./dotenv - echo "${OPENSHIFT_CONSOLE_URL}" > cluster-console-url - echo "${OPENSHIFT_CONSOLE_USERNAME}" > cluster-console-username - echo "${OPENSHIFT_CONSOLE_PASSWORD}" > cluster-console-password - volumeMounts: - - name: data - mountPath: /data - outputs: - artifacts: - - name: cluster-console-url - path: /data/cluster-console-url - archive: - none: {} - - name: cluster-console-username - path: /data/cluster-console-username - archive: - none: {} - - name: cluster-console-password - path: /data/cluster-console-password - archive: - none: {} - - - name: wait - suspend: {} - - - name: destroy - container: - image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} - imagePullPolicy: Always - command: - - entrypoint.sh - args: - - destroy - - '{{ "{{" }}workflow.parameters.name{{ "}}" }}' - env: - - name: GOOGLE_CREDENTIALS - valueFrom: - secretKeyRef: - name: openshift-4-gcp-service-account - key: google-credentials.json - - name: GCP_PROJECT - value : "acs-team-temp-dev" - volumeMounts: - - name: data - mountPath: /data diff --git a/chart/infra-server/templates/secrets.yaml b/chart/infra-server/templates/secrets.yaml index 2147de435..ddf06699f 100644 --- a/chart/infra-server/templates/secrets.yaml +++ b/chart/infra-server/templates/secrets.yaml @@ -46,9 +46,6 @@ data: workflow-openshift-4-demo.yaml: |- {{- tpl (.Files.Get "static/workflow-openshift-4-demo.yaml" ) . | b64enc | nindent 4 }} - workflow-openshift-4-perf-scale.yaml: |- - {{- tpl (.Files.Get "static/workflow-openshift-4-perf-scale.yaml" ) . | b64enc | nindent 4 }} - workflow-eks.yaml: |- {{- tpl (.Files.Get "static/workflow-eks.yaml" ) . | b64enc | nindent 4 }} From 447440301418d88c4e77ce58859594fd38364aae Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Mon, 7 Apr 2025 13:30:43 +0200 Subject: [PATCH 4/9] fix --- chart/infra-server/static/workflow-openshift-4.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/infra-server/static/workflow-openshift-4.yaml b/chart/infra-server/static/workflow-openshift-4.yaml index 5f64bbdb2..d17323418 100644 --- a/chart/infra-server/static/workflow-openshift-4.yaml +++ b/chart/infra-server/static/workflow-openshift-4.yaml @@ -24,7 +24,7 @@ spec: - name: fips-enabled - name: trusted-certs-enabled - name: credentials-mode - - name: keep-failed-cluster + - name: keep-failed-cluster # this not present in openshift-4-perf-scale - name: ssd-storage-class volumeClaimTemplates: - metadata: From 972cfb24a330f35f36e5afbe333372b657fe71e8 Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Wed, 15 Oct 2025 15:40:44 +0200 Subject: [PATCH 5/9] fix perf-scale flavor --- chart/infra-server/static/flavors.yaml | 5 +++++ chart/infra-server/static/test-simulate.yaml | 1 - chart/infra-server/static/workflow-openshift-4.yaml | 2 +- scripts/deploy/helm.sh | 3 ++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/chart/infra-server/static/flavors.yaml b/chart/infra-server/static/flavors.yaml index 5898c3a5b..a95a70578 100644 --- a/chart/infra-server/static/flavors.yaml +++ b/chart/infra-server/static/flavors.yaml @@ -576,6 +576,11 @@ value: false kind: optional + - name: install-hypershift + description: should Hypershift be installed + value: false + kind: optional + - name: trusted-certs-enabled description: Should trusted certificates be created value: false diff --git a/chart/infra-server/static/test-simulate.yaml b/chart/infra-server/static/test-simulate.yaml index f500db383..f1deced94 100644 --- a/chart/infra-server/static/test-simulate.yaml +++ b/chart/infra-server/static/test-simulate.yaml @@ -8,7 +8,6 @@ spec: arguments: parameters: - name: name - value: "" - name: create-delay-seconds value: "" - name: create-outcome diff --git a/chart/infra-server/static/workflow-openshift-4.yaml b/chart/infra-server/static/workflow-openshift-4.yaml index 66aa6fcc5..94e1594cd 100644 --- a/chart/infra-server/static/workflow-openshift-4.yaml +++ b/chart/infra-server/static/workflow-openshift-4.yaml @@ -26,7 +26,7 @@ spec: - name: install-hypershift - name: trusted-certs-enabled - name: credentials-mode - - name: keep-failed-cluster # this not present in openshift-4-perf-scale + - name: keep-failed-cluster - name: ssd-storage-class volumeClaimTemplates: - metadata: diff --git a/scripts/deploy/helm.sh b/scripts/deploy/helm.sh index 073f2d0cb..3f93816fc 100755 --- a/scripts/deploy/helm.sh +++ b/scripts/deploy/helm.sh @@ -14,7 +14,8 @@ TEST_MODE="${TEST_MODE:-false}" SECRETS_PROJECT="acs-team-automation" RELEASE_NAMESPACE="infra" RELEASE_NAME="infra-server" -ARGO_WORKFLOWS_APP_VERSION="v3.6.5" +# TODO: This should be synced with the version in the charts/infra-server/Chart.yaml dependency. +ARGO_WORKFLOWS_APP_VERSION="v3.6.4" check_not_empty() { for V in "$@"; do From fed2c791f68bff442d61ca8e5d7c595ba5a2d03f Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Wed, 15 Oct 2025 17:01:00 +0200 Subject: [PATCH 6/9] fix imagepullbackoff errors for test-qa-demo by using more modern image (cowsay:latest is not compatible with modern GKE), fix pod name logic to support templateRefs --- chart/infra-server/static/test-qa-demo.yaml | 4 ++-- pkg/service/cluster/cluster.go | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/chart/infra-server/static/test-qa-demo.yaml b/chart/infra-server/static/test-qa-demo.yaml index f9c9514ea..f3ec25922 100644 --- a/chart/infra-server/static/test-qa-demo.yaml +++ b/chart/infra-server/static/test-qa-demo.yaml @@ -23,8 +23,8 @@ spec: - name: whalesay container: - image: docker/whalesay:latest - command: [cowsay] + image: busybox:latest + command: [echo] args: - "hello world to: " - '{{ "{{" }}workflow.parameters.name{{ "}}" }}' diff --git a/pkg/service/cluster/cluster.go b/pkg/service/cluster/cluster.go index c69f2dfc4..ecfeda4a8 100644 --- a/pkg/service/cluster/cluster.go +++ b/pkg/service/cluster/cluster.go @@ -733,7 +733,14 @@ func determinePodName(node v1alpha1.NodeStatus) string { parts := strings.Split(node.ID, "-") baseName := strings.Join(parts[:len(parts)-1], "-") randomNumber := parts[len(parts)-1] - return fmt.Sprintf("%s-%s-%s", baseName, node.TemplateName, randomNumber) + + var templateName string + if node.TemplateRef != nil { + templateName = node.TemplateRef.Template + } else { + templateName = node.TemplateName + } + return fmt.Sprintf("%s-%s-%s", baseName, templateName, randomNumber) } func (s *clusterImpl) startSlackCheck() { From 4f2ab1df96e1e527041d32e9969978fc243f4651 Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Thu, 16 Oct 2025 11:34:24 +0200 Subject: [PATCH 7/9] determine ARGO_WORKFLOWS_APP_VERSION based on chart --- Makefile | 6 +++--- scripts/deploy/helm.sh | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index ea64bccd6..3e948eb93 100644 --- a/Makefile +++ b/Makefile @@ -232,11 +232,11 @@ endif .PHONY: helm-dependency-update helm-dependency-update: - @helm dependency update chart/infra-server + @helm dependency update chart/infra-server >&2 create-namespaces: - @kubectl create namespace argo >/dev/null 2>&1 || echo "namespace/argo already exists"; exit 0 - @kubectl create namespace monitoring >/dev/null 2>&1 || echo "namespace/monitoring already exists"; exit 0 + @kubectl create namespace argo >/dev/null 2>&1 || echo "namespace/argo already exists" >&2; exit 0 + @kubectl create namespace monitoring >/dev/null 2>&1 || echo "namespace/monitoring already exists" >&2; exit 0 ## Render template .PHONY: helm-template diff --git a/scripts/deploy/helm.sh b/scripts/deploy/helm.sh index 3f93816fc..c219fa267 100755 --- a/scripts/deploy/helm.sh +++ b/scripts/deploy/helm.sh @@ -14,8 +14,6 @@ TEST_MODE="${TEST_MODE:-false}" SECRETS_PROJECT="acs-team-automation" RELEASE_NAMESPACE="infra" RELEASE_NAME="infra-server" -# TODO: This should be synced with the version in the charts/infra-server/Chart.yaml dependency. -ARGO_WORKFLOWS_APP_VERSION="v3.6.4" check_not_empty() { for V in "$@"; do @@ -28,8 +26,11 @@ check_not_empty() { } install_crds() { + argo_chart_file=$(find "chart/infra-server/charts" -name "argo-workflows-*.tgz" 2>/dev/null | head -1) + ARGO_WORKFLOWS_APP_VERSION="$(tar -xzOf "${argo_chart_file}" argo-workflows/Chart.yaml | yq '.appVersion')" + echo "Using argo-workflows app version: ${ARGO_WORKFLOWS_APP_VERSION}" >&2 kubectl apply --kustomize \ - "https://github.com/argoproj/argo-workflows/manifests/base/crds/minimal?ref=${ARGO_WORKFLOWS_APP_VERSION}" + "https://github.com/argoproj/argo-workflows/manifests/base/crds/minimal?ref=${ARGO_WORKFLOWS_APP_VERSION}" >&2 } template() { From 440b19fb4d953a06b5c90852b190740ada7b2787 Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Thu, 16 Oct 2025 12:36:06 +0200 Subject: [PATCH 8/9] fix yq call --- scripts/deploy/helm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deploy/helm.sh b/scripts/deploy/helm.sh index c219fa267..29b03d6a4 100755 --- a/scripts/deploy/helm.sh +++ b/scripts/deploy/helm.sh @@ -27,7 +27,7 @@ check_not_empty() { install_crds() { argo_chart_file=$(find "chart/infra-server/charts" -name "argo-workflows-*.tgz" 2>/dev/null | head -1) - ARGO_WORKFLOWS_APP_VERSION="$(tar -xzOf "${argo_chart_file}" argo-workflows/Chart.yaml | yq '.appVersion')" + ARGO_WORKFLOWS_APP_VERSION="$(tar -xzOf "${argo_chart_file}" argo-workflows/Chart.yaml | yq eval '.appVersion')" echo "Using argo-workflows app version: ${ARGO_WORKFLOWS_APP_VERSION}" >&2 kubectl apply --kustomize \ "https://github.com/argoproj/argo-workflows/manifests/base/crds/minimal?ref=${ARGO_WORKFLOWS_APP_VERSION}" >&2 From 59654620e4849695bb61d8d715a9a6933ca7a55c Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Thu, 16 Oct 2025 12:50:46 +0200 Subject: [PATCH 9/9] self-review and fixing yq 2nd attempt --- chart/infra-server/argo-values.yaml | 2 +- scripts/deploy/helm.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chart/infra-server/argo-values.yaml b/chart/infra-server/argo-values.yaml index c01ea380e..a0802e4d2 100644 --- a/chart/infra-server/argo-values.yaml +++ b/chart/infra-server/argo-values.yaml @@ -3,7 +3,7 @@ argo-workflows: server: authModes: - server - # We do this separately, because they cannot be installed from the dependant chart. + # We install the CRDs separately, because they cannot be upgraded with Helm through the dependant chart. crds: install: false diff --git a/scripts/deploy/helm.sh b/scripts/deploy/helm.sh index 29b03d6a4..a13f250ed 100755 --- a/scripts/deploy/helm.sh +++ b/scripts/deploy/helm.sh @@ -27,7 +27,7 @@ check_not_empty() { install_crds() { argo_chart_file=$(find "chart/infra-server/charts" -name "argo-workflows-*.tgz" 2>/dev/null | head -1) - ARGO_WORKFLOWS_APP_VERSION="$(tar -xzOf "${argo_chart_file}" argo-workflows/Chart.yaml | yq eval '.appVersion')" + ARGO_WORKFLOWS_APP_VERSION="$(tar -xzOf "${argo_chart_file}" argo-workflows/Chart.yaml | yq eval '.appVersion' -)" echo "Using argo-workflows app version: ${ARGO_WORKFLOWS_APP_VERSION}" >&2 kubectl apply --kustomize \ "https://github.com/argoproj/argo-workflows/manifests/base/crds/minimal?ref=${ARGO_WORKFLOWS_APP_VERSION}" >&2