Skip to content

Commit bef98bb

Browse files
dustman9000dustman9000
committed
SREP-4417: Add AVO operator e2e as Prow periodic with ephemeral cluster
Create reusable step registry components for operator e2e testing: - rosa-operator-install: installs operator via PKO ClusterPackage - rosa-operator-e2e: runs Ginkgo e2e binary against target cluster - rosa-operator-e2e-workflow: provisions ROSA Classic STS, installs operator, runs tests, deprovisions Add AVO as the first consumer with a daily periodic at 6 AM UTC. No osde2e dependency -- runs the Ginkgo binary directly with cluster access via backplane. Jira: SREP-4417
1 parent 049eeb2 commit bef98bb

15 files changed

Lines changed: 429 additions & 0 deletions

‎ci-operator/config/openshift/aws-vpce-operator/openshift-aws-vpce-operator-main__periodics.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
base_images:
2+
rosa-aws-cli:
3+
name: rosa-aws-cli
4+
namespace: ci
5+
tag: latest
6+
build_root:
7+
from_repository: true
8+
images:
9+
items:
10+
- dockerfile_path: test/e2e/Containerfile.prow
11+
to: operator-e2e
12+
resources:
13+
'*':
14+
limits:
15+
memory: 4Gi
16+
requests:
17+
cpu: 100m
18+
memory: 200Mi
19+
tests:
20+
- as: avo-e2e-stage
21+
cron: 0 6 * * *
22+
steps:
23+
cluster_profile: rosa-e2e-01
24+
env:
25+
CHANNEL_GROUP: stable
26+
OCM_LOGIN_ENV: staging
27+
OPERATOR_NAME: aws-vpce-operator
28+
OPERATOR_PKO_IMAGE: "quay.io/redhat-services-prod/openshift/aws-vpce-operator-pko:latest"
29+
OPERATOR_IMAGE: "quay.io/redhat-services-prod/openshift/aws-vpce-operator:latest"
30+
workflow: rosa-operator-e2e-workflow
31+
zz_generated_metadata:
32+
branch: main
33+
org: openshift
34+
repo: aws-vpce-operator
35+
variant: periodics

‎ci-operator/jobs/openshift/aws-vpce-operator/openshift-aws-vpce-operator-main-periodics.yaml‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
periodics:
2+
- agent: kubernetes
3+
cluster: build11
4+
cron: 0 6 * * *
5+
decorate: true
6+
extra_refs:
7+
- base_ref: main
8+
org: openshift
9+
repo: aws-vpce-operator
10+
labels:
11+
ci-operator.openshift.io/cloud: aws
12+
ci-operator.openshift.io/cloud-cluster-profile: rosa-e2e-01
13+
ci-operator.openshift.io/variant: periodics
14+
ci.openshift.io/generator: prowgen
15+
pj-rehearse.openshift.io/can-be-rehearsed: "true"
16+
name: periodic-ci-openshift-aws-vpce-operator-main-periodics-avo-e2e-stage
17+
spec:
18+
containers:
19+
- args:
20+
- --gcs-upload-secret=/secrets/gcs/service-account.json
21+
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
22+
- --lease-server-credentials-file=/etc/boskos/credentials
23+
- --report-credentials-file=/etc/report/credentials
24+
- --target=avo-e2e-stage
25+
- --variant=periodics
26+
command:
27+
- ci-operator
28+
env:
29+
- name: HTTP_SERVER_IP
30+
valueFrom:
31+
fieldRef:
32+
fieldPath: status.podIP
33+
image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
34+
imagePullPolicy: Always
35+
name: ""
36+
ports:
37+
- containerPort: 8080
38+
name: http
39+
resources:
40+
requests:
41+
cpu: 10m
42+
volumeMounts:
43+
- mountPath: /etc/boskos
44+
name: boskos
45+
readOnly: true
46+
- mountPath: /secrets/gcs
47+
name: gcs-credentials
48+
readOnly: true
49+
- mountPath: /secrets/manifest-tool
50+
name: manifest-tool-local-pusher
51+
readOnly: true
52+
- mountPath: /etc/pull-secret
53+
name: pull-secret
54+
readOnly: true
55+
- mountPath: /etc/report
56+
name: result-aggregator
57+
readOnly: true
58+
serviceAccountName: ci-operator
59+
volumes:
60+
- name: boskos
61+
secret:
62+
items:
63+
- key: credentials
64+
path: credentials
65+
secretName: boskos-credentials
66+
- name: manifest-tool-local-pusher
67+
secret:
68+
secretName: manifest-tool-local-pusher
69+
- name: pull-secret
70+
secret:
71+
secretName: registry-pull-credentials
72+
- name: result-aggregator
73+
secret:
74+
secretName: result-aggregator

‎ci-operator/jobs/openshift/aws-vpce-operator/openshift-aws-vpce-operator-main-presubmits.yaml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,61 @@ presubmits:
246246
secret:
247247
secretName: result-aggregator
248248
trigger: (?m)^/test( | .* )lint,?($|\s.*)
249+
- agent: kubernetes
250+
always_run: true
251+
branches:
252+
- ^main$
253+
- ^main-
254+
cluster: build11
255+
context: ci/prow/periodics-images
256+
decorate: true
257+
labels:
258+
ci-operator.openshift.io/variant: periodics
259+
ci.openshift.io/generator: prowgen
260+
pj-rehearse.openshift.io/can-be-rehearsed: "true"
261+
name: pull-ci-openshift-aws-vpce-operator-main-periodics-images
262+
rerun_command: /test periodics-images
263+
spec:
264+
containers:
265+
- args:
266+
- --gcs-upload-secret=/secrets/gcs/service-account.json
267+
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
268+
- --report-credentials-file=/etc/report/credentials
269+
- --target=[images]
270+
- --variant=periodics
271+
command:
272+
- ci-operator
273+
image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
274+
imagePullPolicy: Always
275+
name: ""
276+
resources:
277+
requests:
278+
cpu: 10m
279+
volumeMounts:
280+
- mountPath: /secrets/gcs
281+
name: gcs-credentials
282+
readOnly: true
283+
- mountPath: /secrets/manifest-tool
284+
name: manifest-tool-local-pusher
285+
readOnly: true
286+
- mountPath: /etc/pull-secret
287+
name: pull-secret
288+
readOnly: true
289+
- mountPath: /etc/report
290+
name: result-aggregator
291+
readOnly: true
292+
serviceAccountName: ci-operator
293+
volumes:
294+
- name: manifest-tool-local-pusher
295+
secret:
296+
secretName: manifest-tool-local-pusher
297+
- name: pull-secret
298+
secret:
299+
secretName: registry-pull-credentials
300+
- name: result-aggregator
301+
secret:
302+
secretName: result-aggregator
303+
trigger: (?m)^/test( | .* )periodics-images,?($|\s.*)
249304
- agent: kubernetes
250305
always_run: false
251306
branches:

‎ci-operator/step-registry/rosa/operator/e2e-workflow/OWNERS‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
approvers:
2+
- tiwillia
3+
- dustman9000
4+
- bmeng
5+
- ravitri
6+
reviewers:
7+
- tiwillia
8+
- dustman9000
9+
- bmeng
10+
- ravitri

‎ci-operator/step-registry/rosa/operator/e2e-workflow/rosa-operator-e2e-workflow-workflow.metadata.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"path": "rosa/operator/e2e-workflow/rosa-operator-e2e-workflow-workflow.yaml",
3+
"owners": {
4+
"approvers": ["tiwillia", "dustman9000", "bmeng", "ravitri"],
5+
"reviewers": ["tiwillia", "dustman9000", "bmeng", "ravitri"]
6+
}
7+
}

‎ci-operator/step-registry/rosa/operator/e2e-workflow/rosa-operator-e2e-workflow-workflow.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
workflow:
2+
as: rosa-operator-e2e-workflow
3+
steps:
4+
env:
5+
CHANNEL_GROUP: stable
6+
OPENSHIFT_VERSION: ""
7+
pre:
8+
- chain: rosa-aws-sts-provision
9+
- ref: rosa-cluster-wait-ready-nodes
10+
- ref: rosa-operator-install
11+
test:
12+
- ref: rosa-operator-e2e
13+
post:
14+
- ref: osd-gather-extra
15+
- chain: rosa-aws-sts-deprovision
16+
documentation: |-
17+
Provisions a ROSA Classic STS cluster, installs an operator via PKO,
18+
runs the operator e2e tests, then deprovisions. Designed for operator
19+
CI validation with ephemeral clusters.

‎ci-operator/step-registry/rosa/operator/e2e/OWNERS‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
approvers:
2+
- tiwillia
3+
- dustman9000
4+
- bmeng
5+
- ravitri
6+
reviewers:
7+
- tiwillia
8+
- dustman9000
9+
- bmeng
10+
- ravitri

‎ci-operator/step-registry/rosa/operator/e2e/rosa-operator-e2e-commands.sh‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#!/bin/bash
2+
3+
set -o nounset
4+
set -o errexit
5+
set -o pipefail
6+
7+
trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM
8+
9+
log(){
10+
echo -e "\033[1m$(date "+%d-%m-%YT%H:%M:%S") " "${*}\033[0m" >&2
11+
}
12+
13+
if [[ -z "${OPERATOR_NAME:-}" ]]; then
14+
log "ERROR: OPERATOR_NAME is required"
15+
exit 1
16+
fi
17+
18+
# Get cluster access: prefer shared kubeconfig from provision step,
19+
# fall back to backplane for persistent clusters
20+
if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then
21+
log "Using kubeconfig from provision step"
22+
export KUBECONFIG="${SHARED_DIR}/kubeconfig"
23+
elif [[ -n "${OPERATOR_E2E_CLUSTER_ID:-}" ]]; then
24+
# Log into OCM for backplane access
25+
SSO_CLIENT_ID=$(cat "${CLUSTER_PROFILE_DIR}/sso-client-id" 2>/dev/null || true)
26+
SSO_CLIENT_SECRET=$(cat "${CLUSTER_PROFILE_DIR}/sso-client-secret" 2>/dev/null || true)
27+
OCM_TOKEN=$(cat "${CLUSTER_PROFILE_DIR}/ocm-token" 2>/dev/null || true)
28+
29+
if [[ -n "${SSO_CLIENT_ID}" && -n "${SSO_CLIENT_SECRET}" ]]; then
30+
log "Logging into ${OCM_LOGIN_ENV} with SSO credentials"
31+
ocm login --url "${OCM_LOGIN_ENV}" --client-id "${SSO_CLIENT_ID}" --client-secret "${SSO_CLIENT_SECRET}"
32+
elif [[ -n "${OCM_TOKEN}" ]]; then
33+
log "Logging into ${OCM_LOGIN_ENV} with offline token"
34+
ocm login --url "${OCM_LOGIN_ENV}" --token "${OCM_TOKEN}"
35+
else
36+
log "ERROR: No OCM credentials found in cluster profile"
37+
exit 1
38+
fi
39+
40+
log "Getting kubeconfig for cluster ${OPERATOR_E2E_CLUSTER_ID} via backplane"
41+
ocm backplane login "${OPERATOR_E2E_CLUSTER_ID}"
42+
else
43+
log "ERROR: No cluster access method available (no SHARED_DIR/kubeconfig or OPERATOR_E2E_CLUSTER_ID)"
44+
exit 1
45+
fi
46+
47+
# Verify cluster access
48+
oc whoami
49+
log "Connected to cluster: $(oc whoami --show-server)"
50+
51+
# Run the operator e2e tests
52+
JUNIT_REPORT="${ARTIFACT_DIR}/junit-${OPERATOR_NAME}-e2e.xml"
53+
GINKGO_FLAGS="--ginkgo.junit-report=${JUNIT_REPORT} --ginkgo.v"
54+
55+
if [[ -n "${GINKGO_LABEL_FILTER:-}" ]]; then
56+
GINKGO_FLAGS="${GINKGO_FLAGS} --ginkgo.label-filter=${GINKGO_LABEL_FILTER}"
57+
fi
58+
59+
log "Running ${OPERATOR_NAME} e2e tests..."
60+
/usr/local/bin/e2e.test ${GINKGO_FLAGS} || {
61+
log "Tests failed. JUnit report at ${JUNIT_REPORT}"
62+
exit 1
63+
}
64+
65+
log "Tests passed. JUnit report at ${JUNIT_REPORT}"

‎ci-operator/step-registry/rosa/operator/e2e/rosa-operator-e2e-ref.metadata.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"path": "rosa/operator/e2e/rosa-operator-e2e-ref.yaml",
3+
"owners": {
4+
"approvers": ["tiwillia", "dustman9000", "bmeng", "ravitri"],
5+
"reviewers": ["tiwillia", "dustman9000", "bmeng", "ravitri"]
6+
}
7+
}

‎ci-operator/step-registry/rosa/operator/e2e/rosa-operator-e2e-ref.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
ref:
2+
as: rosa-operator-e2e
3+
from: operator-e2e
4+
grace_period: 10m
5+
commands: rosa-operator-e2e-commands.sh
6+
resources:
7+
requests:
8+
cpu: 100m
9+
memory: 300Mi
10+
timeout: 1h0m0s
11+
env:
12+
- name: OPERATOR_NAME
13+
default: ""
14+
documentation: Short name of the operator (e.g. aws-vpce-operator).
15+
- name: OPERATOR_E2E_CLUSTER_ID
16+
default: ""
17+
documentation: OCM cluster ID to test against. For MC/SC-resident operators, use the MC or SC cluster ID directly.
18+
- name: OCM_LOGIN_ENV
19+
default: "production"
20+
documentation: OCM environment where the target cluster is registered. MCs and SCs are registered in production.
21+
- name: GINKGO_LABEL_FILTER
22+
default: ""
23+
documentation: Optional Ginkgo label filter expression.
24+
documentation: |-
25+
Runs an operator e2e test binary against an existing ROSA cluster.
26+
Uses OCM credentials from the cluster profile to get cluster access
27+
via backplane. For MC/SC-resident operators (AVO, RMO, etc.), target
28+
the MC or SC directly. The e2e binary must be baked into the
29+
operator-e2e image at /usr/local/bin/e2e.test.

0 commit comments

Comments
 (0)