[DX-3553] add CRE upgrade job to nightly and release (#21666)

Tofel · web-flow · commit 2f0c8e9c7885 · 2026-03-26T17:14:30.000Z
* add CRE upgrade pipeline

* bump CTF

* pass some buildcmd

* save Docker logs

* clean up + uncomment df1 compat tests

* exclude rc and beta refs from upgrade tests

* support passing custom refs

* add upgrade tests to build publish pipeline
diff --git a/.github/workflows/build-publish.yml b/.github/workflows/build-publish.yml
@@ -205,6 +205,45 @@ jobs:
           products: |
             ccip-prereleases-prod-testnet
 
+  # Upgrade compatibility tests
+  df1-compat:
+    uses: ./.github/workflows/devenv-compat.yml
+    name: Data Feeds v1 Upgrade Test
+    needs: [docker-core]
+    with:
+      buildcmd: "just cli"
+      envcmd: "cl u env.toml,products/ocr2/basic.toml"
+      testcmd: "cl test ocr2 TestSmoke/rounds"
+      upgrade-nodes: "3"
+      node-name-template: "don-node%d"
+      versions-back: "2"
+      # uncomment once we have tested this pipeline with beta/rc releases
+      # exclude-refs: "beta,rc"
+      working-directory: "devenv"
+      logs-directory: "./devenv/tests/ocr2/logs"
+      ctf-log-level: "info"
+      strip-image-suffix: "v"
+    secrets: inherit
+
+  cre-compat:
+    uses: ./.github/workflows/devenv-compat.yml
+    name: CRE Upgrade Test
+    needs: [docker-core]
+    with:
+      buildcmd: "echo nothing-to-build" # added to prevent `just cli` from being used (default buildcmd if not provided)
+      envcmd: "go run -C ../../core/scripts/cre/environment . env start"
+      testcmd: "go test -v -run Test_Upgrade_Suite ./smoke/cre"
+      upgrade-nodes: "3"
+      node-name-template: "workflow-node%d"
+      versions-back: "2"
+      # uncomment once we have tested this pipeline with beta/rc releases
+      # exclude-refs: "beta,rc"
+      working-directory: "./system-tests/tests"
+      logs-directory: "./system-tests/tests/smoke/cre/logs"
+      ctf-log-level: "info"
+      strip-image-suffix: "v"
+    secrets: inherit
+
   # Notify Slack channel for new git tags associated with pre-releases.
   # Final release notifications originate from the release coordinator repo.
   slack-notify:
diff --git a/.github/workflows/devenv-compat.yml b/.github/workflows/devenv-compat.yml
@@ -20,6 +20,10 @@ on:
         description: "Git refs to test, should be used only for tool testing purposes. In all other cases refs are detected automatically either from Git or RANE SOT"
         required: false
         type: string
+      exclude-refs:
+        description: "Patterns to exclude specific refs (e.g., beta,rc,v0,v1)"
+        required: false
+        type: string
       buildcmd:
         description: "Build command to run"
         required: true
@@ -83,6 +87,7 @@ jobs:
       PRODUCT: ${{ inputs.product }}
       NO_GIT_ROLLBACK: ${{ inputs.no-git-rollback && '--no-git-rollback' || '' }} # Convert boolean to flag string
       REFS: ${{ inputs.refs }}
+      EXCLUDE_REFS: ${{ inputs.exclude-refs }}
       BUILD_CMD: ${{ inputs.buildcmd }}
       ENV_CMD: ${{ inputs.envcmd }}
       TEST_CMD: ${{ inputs.testcmd }}
@@ -175,13 +180,18 @@ jobs:
           if [ -n "${{ env.REFS }}" ]; then
             REFS_ARGS="--refs ${{ env.REFS }}"
           fi
+          EXCLUDE_REFS_ARGS=""
+          if [ -n "${{ env.EXCLUDE_REFS }}" ]; then
+            EXCLUDE_REFS_ARGS="--exclude-refs ${{ env.EXCLUDE_REFS }}"
+          fi
           ./bin/ctf compat backward \
             --registry "${REGISTRY}" \
             --buildcmd "${BUILD_CMD}" \
             --envcmd "${ENV_CMD}" \
             --testcmd "${TEST_CMD}" \
             --product "${PRODUCT}" \
             ${REFS_ARGS} \
+            ${EXCLUDE_REFS_ARGS} \
             --upgrade-nodes "${UPGRADE_NODES}" \
             --don_nodes "${DON_NODES}" \
             --node-name-template "${NODE_NAME_TEMPLATE}" \
diff --git a/.github/workflows/devenv-nightly-compat.yml b/.github/workflows/devenv-nightly-compat.yml
@@ -4,6 +4,11 @@ on:
   schedule:
     - cron: "0 6 * * *" # Run daily at 6 AM
   workflow_dispatch:
+    inputs:
+      refs:
+        description: "Git refs to test, should be used only if automatically detected refs are not enough"
+        required: false
+        type: string
 
 permissions:
   id-token: write
@@ -14,7 +19,7 @@ jobs:
   # CREcluster compatibility and upgrade test
   # TODO
 
-  # Data Feeds v1 upgrade test 
+  # Data Feeds v1 upgrade test
   # this verifies that all the versions existing in RANE SOT for Data Feeds v1 can be upgraded to the latest version
   # TODO: disabled for now until we have connectivity
   # df1-upgrade:
@@ -40,6 +45,7 @@ jobs:
   # use it as an example project
   df1-compat:
     uses: ./.github/workflows/devenv-compat.yml
+    name: Data Feeds v1 Upgrade Test
     with:
       # bash command to build your environment, omit if you are using 'devenv'
       buildcmd: "just cli"
@@ -49,10 +55,14 @@ jobs:
       testcmd: "cl test ocr2 TestSmoke/rounds"
       # number of CL nodes to upgrade with new version
       upgrade-nodes: "3"
-      # cl node Docker cocntainer name template, omit if you are using 'devenv'
+      # cl node Docker container name template, omit if you are using 'devenv'
       node-name-template: "don-node%d"
       # previous versions to test (sorted by tags, SemVer)
       versions-back: "2"
+      # patterns to exclude specific refs
+      exclude-refs: "beta,rc"
+      # Git refs to test, should be used only for tool testing purposes.
+      refs: ${{ github.event_name == 'workflow_dispatch' && inputs.refs || '' }}
       # directory with your developer environment to run buildcmd and envcmd
       working-directory: "devenv"
       # directory for executing your tests, to run testcmd
@@ -62,3 +72,33 @@ jobs:
       # this is needed to remove 'v' from Git tags to match SDLC ECR tags
       strip-image-suffix: "v"
     secrets: inherit
+
+  cre-compat:
+    uses: ./.github/workflows/devenv-compat.yml
+    name: CRE Upgrade Test
+    with:
+      # bash command to build your environment, omit if you are using 'devenv'
+      buildcmd: "echo nothing-to-build"
+      # bash command to spin up your environment and product orchestration
+      envcmd: "go run -C ../../core/scripts/cre/environment . env start"
+      # bash command to run the test verifying your product
+      testcmd: "go test -v -run Test_Upgrade_Suite ./smoke/cre"
+      # number of CL nodes to upgrade with new version
+      upgrade-nodes: "3"
+      # cl node Docker container name template, omit if you are using 'devenv'
+      node-name-template: "workflow-node%d"
+      # previous versions to test (sorted by tags, SemVer)
+      versions-back: "2"
+      # patterns to exclude specific refs
+      exclude-refs: "beta,rc"
+      # Git refs to test, should be used only for tool testing purposes.
+      refs: ${{ github.event_name == 'workflow_dispatch' && inputs.refs || '' }}
+      # directory with your developer environment to run buildcmd and envcmd
+      working-directory: "./system-tests/tests"
+      # directory for executing your tests, to run testcmd
+      logs-directory: "./system-tests/tests/smoke/cre/logs"
+      # CTF log level
+      ctf-log-level: "info"
+      # this is needed to remove 'v' from Git tags to match SDLC ECR tags
+      strip-image-suffix: "v"
+    secrets: inherit
diff --git a/core/scripts/go.mod b/core/scripts/go.mod
@@ -54,7 +54,7 @@ require (
 	github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260119171452-39c98c3b33cd
 	github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260320153346-314ec8dbe5a4
 	github.com/smartcontractkit/chainlink-protos/job-distributor v0.18.0
-	github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.5
+	github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.8
 	github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.20
 	github.com/smartcontractkit/chainlink-testing-framework/lib v1.54.5
 	github.com/smartcontractkit/chainlink-testing-framework/seth v1.51.5
diff --git a/core/scripts/go.sum b/core/scripts/go.sum
diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod
@@ -41,7 +41,7 @@ require (
 	github.com/smartcontractkit/chainlink-protos/job-distributor v0.18.0
 	github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260217043601-5cc966896c4f
 	github.com/smartcontractkit/chainlink-solana v1.1.2-0.20260325152920-167c7a34804c
-	github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.5
+	github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.8
 	github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.15
 	github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.10.0
 	github.com/smartcontractkit/chainlink-testing-framework/lib v1.54.5
diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum
diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod
@@ -66,7 +66,7 @@ require (
 	github.com/smartcontractkit/chainlink-protos/job-distributor v0.18.0
 	github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260128151123-605e9540b706
 	github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260217043601-5cc966896c4f
-	github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.5
+	github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.8
 	github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.10.0
 	github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.7
 	github.com/smartcontractkit/chainlink-testing-framework/lib v1.54.5
diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum
