diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml index dc9670ad9..c9f2009bc 100644 --- a/.github/workflows/PR.yaml +++ b/.github/workflows/PR.yaml @@ -34,7 +34,7 @@ jobs: with: flavor: gke-default name: infra-pr-${{ github.event.pull_request.number }} - args: machine-type=e2-medium,nodes=3,gcp-image-type=ubuntu_containerd + args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd lifespan: ${{ github.actor == 'dependabot[bot]' && '1h' || '24h' }} wait: true token: ${{ secrets.INFRA_TOKEN }} diff --git a/.github/workflows/smoke-test.yaml b/.github/workflows/smoke-test.yaml index 131f85517..21ec6a62c 100644 --- a/.github/workflows/smoke-test.yaml +++ b/.github/workflows/smoke-test.yaml @@ -46,10 +46,9 @@ jobs: {name: "ocp-4", "args": "", "uniqueness": "g"}, {name: "ocp-4-demo", "args": "", "uniqueness": "h"}, {name: "ocp-4-perf-scale", "args": "", "uniqueness": "i"}, - {name: "ocp-3", "args": "", "uniqueness": "j"}, {name: "osd-on-aws", "args": "", "uniqueness": "k"}, {name: "osd-on-gcp", "args": "", "uniqueness": "l"}, - {name: "qa-demo", "args": "main-image=quay.io/rhacs-eng/main:4.3.4", "uniqueness": "m"}, + {name: "qa-demo", "args": "main-image=quay.io/rhacs-eng/main:4.7.0", "uniqueness": "m"}, {name: "rosa", "args": "", "uniqueness": "n"}, {name: "rosahcp", "args": "", "uniqueness": "o"}, ] diff --git a/chart/infra-server/static/flavors.yaml b/chart/infra-server/static/flavors.yaml index cddd7572e..387b5fe47 100644 --- a/chart/infra-server/static/flavors.yaml +++ b/chart/infra-server/static/flavors.yaml @@ -1243,7 +1243,7 @@ value: "" - name: create-delay-seconds description: how long to spend in create - value: "10" + value: "0" kind: optional - name: create-outcome description: success or fail @@ -1251,12 +1251,16 @@ kind: optional - name: destroy-delay-seconds description: how long to spend in destroy - value: "10" + value: "0" kind: optional - name: destroy-outcome description: success or fail value: success kind: optional + - name: test-gcs + description: whether to upload or delete a test artifact in GCS + value: "false" + kind: optional ################ # Test Janitor # diff --git a/chart/infra-server/static/test-connect-artifact.yaml b/chart/infra-server/static/test-connect-artifact.yaml index 52d7983bc..8e09ef3d8 100644 --- a/chart/infra-server/static/test-connect-artifact.yaml +++ b/chart/infra-server/static/test-connect-artifact.yaml @@ -4,6 +4,7 @@ metadata: generateName: test-connect-artifact- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -16,6 +17,8 @@ spec: template: create - - name: wait template: wait + - name: stop + steps: - - name: destroy template: destroy diff --git a/chart/infra-server/static/test-gke-lite.yaml b/chart/infra-server/static/test-gke-lite.yaml index 6e8a1cd8d..a7b1e00ed 100644 --- a/chart/infra-server/static/test-gke-lite.yaml +++ b/chart/infra-server/static/test-gke-lite.yaml @@ -4,6 +4,7 @@ metadata: generateName: gke-lite- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -30,12 +31,14 @@ spec: template: create - - name: wait template: wait + - name: stop + steps: - - name: destroy template: destroy arguments: parameters: - name: name - value: '{{ "{{" }}steps.create.outputs.parameters.cluster_name{{ "}}" }}' + value: '{{ "{{" }}workflow.parameters.name{{ "}}" }}' - name: create activeDeadlineSeconds: 3600 diff --git a/chart/infra-server/static/test-qa-demo.yaml b/chart/infra-server/static/test-qa-demo.yaml index ff9df6edf..f9c9514ea 100644 --- a/chart/infra-server/static/test-qa-demo.yaml +++ b/chart/infra-server/static/test-qa-demo.yaml @@ -4,6 +4,7 @@ metadata: generateName: test-qa-demo- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -15,6 +16,8 @@ spec: template: whalesay - - name: wait template: wait + - name: stop + steps: - - name: destroy template: whalesay diff --git a/chart/infra-server/static/test-simulate.yaml b/chart/infra-server/static/test-simulate.yaml index 2e7f7cd41..ba7c5f311 100644 --- a/chart/infra-server/static/test-simulate.yaml +++ b/chart/infra-server/static/test-simulate.yaml @@ -4,6 +4,7 @@ metadata: generateName: simulate- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -15,6 +16,13 @@ spec: value: "" - name: destroy-outcome value: "" + - name: test-gcs + value: "" + + volumes: + - name: credentials + secret: + secretName: google-credentials templates: - name: start @@ -27,8 +35,12 @@ spec: value: '{{ "{{" }}workflow.parameters.create-delay-seconds{{ "}}" }}' - name: outcome value: '{{ "{{" }}workflow.parameters.create-outcome{{ "}}" }}' + - name: test-gcs + value: '{{ "{{"}}workflow.parameters.test-gcs{{ "}}" }}' - - name: wait template: wait + - name: stop + steps: - - name: destroy template: simulate arguments: @@ -37,24 +49,62 @@ spec: value: '{{ "{{" }}workflow.parameters.destroy-delay-seconds{{ "}}" }}' - name: outcome value: '{{ "{{" }}workflow.parameters.destroy-outcome{{ "}}" }}' + - name: test-gcs + value: '{{ "{{"}}workflow.parameters.test-gcs{{ "}}" }}' - name: simulate inputs: parameters: - name: delay-seconds - name: outcome + - name: test-gcs script: - image: debian:9.4 + image: gcr.io/google.com/cloudsdktool/google-cloud-cli:stable command: [bash] source: | set -x - start=0 - while sleep 1; do - if [[ $((start++)) -ge {{ "{{" }}inputs.parameters.delay-seconds{{ "}}" }} ]]; then - break + + delay() { + start=0 + while sleep 1; do + if [[ $((start++)) -ge {{ "{{" }}inputs.parameters.delay-seconds{{ "}}" }} ]]; then + break + fi + done + } + + upload_or_delete_gcs_object() { + gcloud auth activate-service-account --key-file /tmp/google-credentials.json + gcloud config set core/disable_prompts True + + BUCKET_NAME="infra-e2e-upload-test" + FILE="{{ "{{" }}workflow.name{{ "}}" }}" + touch "${FILE}" + + DESTINATION="gs://${BUCKET_NAME}/${FILE}" + + if gsutil -q stat "${DESTINATION}"; then + echo "File exists. Deleting..." + gsutil rm "${DESTINATION}" + else + echo "File does not exist. Proceeding to upload." + gsutil cp "${FILE}" "${DESTINATION}" fi - done + } + + if [[ "{{ "{{" }}inputs.parameters.test-gcs{{ "}}" }}" == "true" ]]; then + upload_or_delete_gcs_object + fi + + if [[ {{ "{{" }}inputs.parameters.delay-seconds{{ "}}" }} -gt 0 ]]; then + delay + fi + [[ "{{ "{{" }}inputs.parameters.outcome{{ "}}" }}" == "success" ]] || exit 1 + volumeMounts: + - name: credentials + mountPath: /tmp + - name: wait suspend: {} diff --git a/chart/infra-server/static/test-url-artifact.yaml b/chart/infra-server/static/test-url-artifact.yaml index 2829daf99..a763025bf 100644 --- a/chart/infra-server/static/test-url-artifact.yaml +++ b/chart/infra-server/static/test-url-artifact.yaml @@ -4,6 +4,7 @@ metadata: generateName: test-url-artifact- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -16,6 +17,8 @@ spec: template: create - - name: wait template: wait + - name: stop + steps: - - name: destroy template: destroy diff --git a/chart/infra-server/static/workflow-aks.yaml b/chart/infra-server/static/workflow-aks.yaml index e3aa81710..1b99c1d8b 100644 --- a/chart/infra-server/static/workflow-aks.yaml +++ b/chart/infra-server/static/workflow-aks.yaml @@ -4,6 +4,7 @@ metadata: generateName: aks- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -27,6 +28,8 @@ spec: - - name: wait template: wait + - name: stop + steps: - - name: destroy template: destroy diff --git a/chart/infra-server/static/workflow-eks.yaml b/chart/infra-server/static/workflow-eks.yaml index e8a073cc2..acb4b88f4 100644 --- a/chart/infra-server/static/workflow-eks.yaml +++ b/chart/infra-server/static/workflow-eks.yaml @@ -4,6 +4,7 @@ metadata: generateName: eks- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -27,6 +28,8 @@ spec: - - name: wait template: wait + - name: stop + steps: - - name: destroy template: destroy diff --git a/chart/infra-server/static/workflow-gke-default.yaml b/chart/infra-server/static/workflow-gke-default.yaml index 6dd1d75dc..b869b874a 100644 --- a/chart/infra-server/static/workflow-gke-default.yaml +++ b/chart/infra-server/static/workflow-gke-default.yaml @@ -4,6 +4,7 @@ metadata: generateName: gke-default- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -30,12 +31,15 @@ spec: template: create - - name: wait template: wait + + - name: stop + steps: - - name: destroy template: destroy arguments: parameters: - name: name - value: '{{ "{{" }}steps.create.outputs.parameters.cluster_name{{ "}}" }}' + value: '{{ "{{" }}workflow.parameters.name{{ "}}" }}' - name: create activeDeadlineSeconds: 3600 diff --git a/chart/infra-server/static/workflow-openshift-4-demo.yaml b/chart/infra-server/static/workflow-openshift-4-demo.yaml index 4f9557997..76181b3f2 100644 --- a/chart/infra-server/static/workflow-openshift-4-demo.yaml +++ b/chart/infra-server/static/workflow-openshift-4-demo.yaml @@ -4,6 +4,7 @@ metadata: generateName: openshift-4-demo- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -74,6 +75,8 @@ spec: - - name: wait template: wait + - name: stop + steps: - - name: destroy template: destroy diff --git a/chart/infra-server/static/workflow-openshift-4-perf-scale.yaml b/chart/infra-server/static/workflow-openshift-4-perf-scale.yaml index 3a0bdaaa6..707d16f16 100644 --- a/chart/infra-server/static/workflow-openshift-4-perf-scale.yaml +++ b/chart/infra-server/static/workflow-openshift-4-perf-scale.yaml @@ -4,6 +4,7 @@ metadata: generateName: openshift-4-perf-scale- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -50,6 +51,8 @@ spec: - - name: wait template: wait + - name: stop + steps: - - name: destroy template: destroy diff --git a/chart/infra-server/static/workflow-openshift-4.yaml b/chart/infra-server/static/workflow-openshift-4.yaml index 5f64bbdb2..0b3ef3b30 100644 --- a/chart/infra-server/static/workflow-openshift-4.yaml +++ b/chart/infra-server/static/workflow-openshift-4.yaml @@ -4,6 +4,7 @@ metadata: generateName: openshift-4- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -51,6 +52,8 @@ spec: - - name: wait template: wait + - name: stop + steps: - - name: destroy template: destroy diff --git a/chart/infra-server/static/workflow-openshift-aro.yaml b/chart/infra-server/static/workflow-openshift-aro.yaml index e8d9088be..4a9092a13 100644 --- a/chart/infra-server/static/workflow-openshift-aro.yaml +++ b/chart/infra-server/static/workflow-openshift-aro.yaml @@ -4,6 +4,7 @@ metadata: generateName: aro- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -32,6 +33,8 @@ spec: - - name: wait template: wait + - name: stop + steps: - - name: destroy template: destroy diff --git a/chart/infra-server/static/workflow-openshift-ibmroks.yaml b/chart/infra-server/static/workflow-openshift-ibmroks.yaml index 8d73babab..4b62b1a51 100644 --- a/chart/infra-server/static/workflow-openshift-ibmroks.yaml +++ b/chart/infra-server/static/workflow-openshift-ibmroks.yaml @@ -4,6 +4,7 @@ metadata: generateName: roks- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -39,6 +40,8 @@ spec: - - name: wait template: wait + - name: stop + steps: - - name: destroy template: destroy diff --git a/chart/infra-server/static/workflow-openshift-rosa-hcp.yaml b/chart/infra-server/static/workflow-openshift-rosa-hcp.yaml index 5b1c7e56a..fdfbfde0b 100644 --- a/chart/infra-server/static/workflow-openshift-rosa-hcp.yaml +++ b/chart/infra-server/static/workflow-openshift-rosa-hcp.yaml @@ -4,6 +4,7 @@ metadata: generateName: rosa- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -40,6 +41,8 @@ spec: - - name: wait template: wait + - name: stop + steps: - - name: destroy template: destroy diff --git a/chart/infra-server/static/workflow-openshift-rosa.yaml b/chart/infra-server/static/workflow-openshift-rosa.yaml index ad07d6a0d..8ca639b4c 100644 --- a/chart/infra-server/static/workflow-openshift-rosa.yaml +++ b/chart/infra-server/static/workflow-openshift-rosa.yaml @@ -4,6 +4,7 @@ metadata: generateName: rosa- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -34,6 +35,8 @@ spec: - - name: wait template: wait + - name: stop + steps: - - name: destroy template: destroy diff --git a/chart/infra-server/static/workflow-osd-aws.yaml b/chart/infra-server/static/workflow-osd-aws.yaml index bfcacbb29..b12bd94ff 100644 --- a/chart/infra-server/static/workflow-osd-aws.yaml +++ b/chart/infra-server/static/workflow-osd-aws.yaml @@ -4,6 +4,7 @@ metadata: generateName: rosa- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -34,6 +35,8 @@ spec: - - name: wait template: wait + - name: stop + steps: - - name: destroy template: destroy diff --git a/chart/infra-server/static/workflow-osd-gcp.yaml b/chart/infra-server/static/workflow-osd-gcp.yaml index 335d7e2b2..0107bbfa3 100644 --- a/chart/infra-server/static/workflow-osd-gcp.yaml +++ b/chart/infra-server/static/workflow-osd-gcp.yaml @@ -4,6 +4,7 @@ metadata: generateName: rosa- spec: entrypoint: start + onExit: stop arguments: parameters: - name: name @@ -33,7 +34,8 @@ spec: - - name: wait template: wait - + - name: stop + steps: - - name: destroy template: destroy diff --git a/test/e2e/cluster/lifecycle_test.go b/test/e2e/cluster/lifecycle_test.go index 96e898ef6..31451bae4 100644 --- a/test/e2e/cluster/lifecycle_test.go +++ b/test/e2e/cluster/lifecycle_test.go @@ -4,6 +4,7 @@ package cluster_test import ( + "context" "testing" "time" @@ -15,29 +16,40 @@ import ( func TestClusterCanRunThroughStandardLifecycle(t *testing.T) { utils.CheckContext() + ctx := context.Background() + clusterID, err := mock.InfractlCreateCluster( "test-simulate", utils.GetUniqueClusterName("standard"), "--lifespan=10s", + "--arg=test-gcs=true", ) assert.NoError(t, err) assert.NotEmpty(t, clusterID) utils.AssertStatusBecomes(t, clusterID, "CREATING") utils.AssertStatusBecomes(t, clusterID, "READY") + exists, err := utils.CheckGCSObjectExists(ctx, clusterID) + assert.NoError(t, err) + assert.True(t, exists) utils.AssertStatusBecomes(t, clusterID, "DESTROYING") utils.AssertStatusBecomes(t, clusterID, "FINISHED") + utils.CheckGCSObjectEventuallyDeleted(ctx, t, clusterID) } func TestClusterCanFailInCreate(t *testing.T) { utils.CheckContext() + ctx := context.Background() + clusterID, err := mock.InfractlCreateCluster( "test-simulate", utils.GetUniqueClusterName("create-fails"), "--lifespan=30s", "--arg=create-outcome=fail", + "--arg=test-gcs=true", ) assert.NoError(t, err) assert.NotEmpty(t, clusterID) utils.AssertStatusBecomes(t, clusterID, "CREATING") utils.AssertStatusBecomes(t, clusterID, "FAILED") + utils.CheckGCSObjectEventuallyDeleted(ctx, t, clusterID) } func TestClusterCanFailInDestroy(t *testing.T) { @@ -57,9 +69,12 @@ func TestClusterCanFailInDestroy(t *testing.T) { func TestClusterCanBeDeleted(t *testing.T) { utils.CheckContext() + ctx := context.Background() + clusterID, err := mock.InfractlCreateCluster( "test-simulate", utils.GetUniqueClusterName("for-deletion"), "--lifespan=5m", + "--arg=test-gcs=true", ) assert.NoError(t, err) assert.NotEmpty(t, clusterID) @@ -71,13 +86,17 @@ func TestClusterCanBeDeleted(t *testing.T) { assert.NoError(t, err) utils.AssertStatusBecomes(t, clusterID, "DESTROYING") utils.AssertStatusBecomes(t, clusterID, "FINISHED") + utils.CheckGCSObjectEventuallyDeleted(ctx, t, clusterID) } func TestClusterCanExpireByChangingLifespan(t *testing.T) { utils.CheckContext() + ctx := context.Background() + clusterID, err := mock.InfractlCreateCluster( "test-simulate", utils.GetUniqueClusterName("for-expire"), "--lifespan=5m", + "--arg=test-gcs=true", ) assert.NoError(t, err) assert.NotEmpty(t, clusterID) @@ -89,6 +108,9 @@ func TestClusterCanExpireByChangingLifespan(t *testing.T) { assert.NoError(t, err) utils.AssertStatusBecomes(t, clusterID, "DESTROYING") utils.AssertStatusBecomes(t, clusterID, "FINISHED") + exists, err := utils.CheckGCSObjectExists(ctx, clusterID) + assert.NoError(t, err) + assert.False(t, exists) } func TestClusterCanBeCreatedWithAliasFlavor(t *testing.T) { diff --git a/test/utils/gcs.go b/test/utils/gcs.go new file mode 100644 index 000000000..70bc55f52 --- /dev/null +++ b/test/utils/gcs.go @@ -0,0 +1,52 @@ +package utils + +import ( + "context" + "fmt" + "log" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/stretchr/testify/assert" + "google.golang.org/api/iterator" +) + +const bucketName = "infra-e2e-upload-test" + +// CheckGCSObjectExists confirms that an object exists in the GCS bucket. +func CheckGCSObjectExists(ctx context.Context, clusterID string) (bool, error) { + client, err := storage.NewClient(ctx) + if err != nil { + return false, err + } + defer client.Close() + + query := &storage.Query{Prefix: clusterID} + it := client.Bucket(bucketName).Objects(ctx, query) + _, err = it.Next() + if err == iterator.Done { + return false, nil + } + if err != nil { + return false, fmt.Errorf("error finding file for prefix (%s): %v", clusterID, err) + } + + return true, nil +} + +// CheckGCSObjectEventuallyDeleted confirms that an object is eventually deletion from the GCS bucket. +func CheckGCSObjectEventuallyDeleted(ctx context.Context, t *testing.T, clusterID string) { + tick := 1 * time.Second + conditionMet := func() bool { + exists, err := CheckGCSObjectExists(ctx, clusterID) + if err != nil { + log.Printf("error when looking for object: %v", err) + return false + } + + return !exists + } + + assert.Eventually(t, conditionMet, defaultTimeout, tick) +}