Split bwc-tests to bwc-rolling-upgrade and bwc-full-restart (opensearch-project#4716) (opensearch-project#4718)

* Split bwc-tests to bwc-rolling-upgrade and bwc-full-restart (opensearch-project#4716)

Signed-off-by: Lantao Jin <ltjin@amazon.com>
(cherry picked from commit a9e5d39)

* resolve conflicts

Signed-off-by: Lantao Jin <ltjin@amazon.com>

---------

Signed-off-by: Lantao Jin <ltjin@amazon.com>

Author: LantaoJin

.github/workflows/sql-test-and-build-workflow.yml

@@ -182,7 +182,7 @@ jobs:
             doctest/build/testclusters/docTestCluster-0/logs/*
             integ-test/build/testclusters/*/logs/*
 
-  bwc-tests:
+  bwc-tests-rolling-upgrade:
     needs: Get-CI-Image-Tag
     runs-on: ubuntu-latest
     strategy:
@@ -204,10 +204,56 @@ jobs:
           distribution: 'temurin'
           java-version: ${{ matrix.java }}
 
-      - name: Run backward compatibility tests
+      - name: Run backward compatibility tests in rolling upgrade
         run: |
           chown -R 1000:1000 `pwd`
-          su `id -un 1000` -c "./scripts/bwctest.sh"
+          su `id -un 1000` -c "./scripts/bwctest-rolling-upgrade.sh"
+
+      - name: Upload test reports
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        continue-on-error: true
+        with:
+          name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc
+          path: |
+            sql/build/reports/**
+            ppl/build/reports/**
+            core/build/reports/**
+            common/build/reports/**
+            opensearch/build/reports/**
+            integ-test/build/reports/**
+            protocol/build/reports/**
+            legacy/build/reports/**
+            plugin/build/reports/**
+            doctest/build/testclusters/docTestCluster-0/logs/*
+            integ-test/build/testclusters/*/logs/*
+
+  bwc-tests-full-restart:
+    needs: Get-CI-Image-Tag
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [11, 17, 21]
+    container:
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
+
+    steps:
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+      - uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+
+      - name: Run backward compatibility tests in full restart
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./scripts/bwctest-full-restart.sh"
 
       - name: Upload test reports
         if: ${{ always() }}

integ-test/build.gradle

@@ -582,83 +582,80 @@ String bwcRemoteFileRoot = "https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/
 String bwcRemoteFileSqlPlugin = bwcRemoteFileRoot + bwcSqlPlugin
 String bwcRemoteFileJSPlugin = bwcRemoteFileRoot + bwcJSPlugin
 
-2.times { i ->
-    testClusters {
-        "${baseName}$i" {
-            testDistribution = "ARCHIVE"
-            numberOfNodes = 3
-            if (bwcBundleTest) {
-                versions = ["1.3.2", currentVersion]
-                nodes.each { node ->
-                    node.extraConfigFile("kirk.pem", file("src/test/resources/security/kirk.pem"))
-                    node.extraConfigFile("kirk-key.pem", file("src/test/resources/security/kirk-key.pem"))
-                    node.extraConfigFile("esnode.pem", file("src/test/resources/security/esnode.pem"))
-                    node.extraConfigFile("esnode-key.pem", file("src/test/resources/security/esnode-key.pem"))
-                    node.extraConfigFile("root-ca.pem", file("src/test/resources/security/root-ca.pem"))
-                    node.setting("plugins.security.disabled", "true")
-                    node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
-                    node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
-                    node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
-                    node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
-                    node.setting("plugins.security.ssl.http.enabled", "true")
-                    node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
-                    node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
-                    node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
-                    node.setting("plugins.security.allow_unsafe_democertificates", "true")
-                    node.setting("plugins.security.allow_default_init_securityindex", "true")
-                    node.setting("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,L=test,C=de")
-                    node.setting("plugins.security.audit.type", "internal_elasticsearch")
-                    node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
-                    node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")
-                    node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]")
-                    node.setting("plugins.security.system_indices.enabled", "true")
-                }
-            } else {
-                versions = ["1.1.0", opensearch_version]
-                plugin(provider(new Callable<RegularFile>(){
-                    @Override
-                    RegularFile call() throws Exception {
-                        return new RegularFile() {
-                            @Override
-                            File getAsFile() {
-                                if (new File("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion").exists()) {
-                                    project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion"))
-                                }
-                                project.mkdir bwcJSPluginPath + bwcVersion
-                                ant.get(src: bwcRemoteFileJSPlugin,
-                                        dest: bwcJSPluginPath + bwcVersion,
-                                        httpusecaches: false)
-                                return fileTree(bwcJSPluginPath + bwcVersion).getSingleFile()
+testClusters {
+    "${baseName}" {
+        testDistribution = "ARCHIVE"
+        numberOfNodes = 3
+        if (bwcBundleTest) {
+            versions = ["1.3.2", currentVersion]
+            nodes.each { node ->
+                node.extraConfigFile("kirk.pem", file("src/test/resources/security/kirk.pem"))
+                node.extraConfigFile("kirk-key.pem", file("src/test/resources/security/kirk-key.pem"))
+                node.extraConfigFile("esnode.pem", file("src/test/resources/security/esnode.pem"))
+                node.extraConfigFile("esnode-key.pem", file("src/test/resources/security/esnode-key.pem"))
+                node.extraConfigFile("root-ca.pem", file("src/test/resources/security/root-ca.pem"))
+                node.setting("plugins.security.disabled", "true")
+                node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
+                node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
+                node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
+                node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
+                node.setting("plugins.security.ssl.http.enabled", "true")
+                node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
+                node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
+                node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
+                node.setting("plugins.security.allow_unsafe_democertificates", "true")
+                node.setting("plugins.security.allow_default_init_securityindex", "true")
+                node.setting("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,L=test,C=de")
+                node.setting("plugins.security.audit.type", "internal_elasticsearch")
+                node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
+                node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")
+                node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]")
+                node.setting("plugins.security.system_indices.enabled", "true")
+            }
+        } else {
+            versions = ["1.1.0", opensearch_version]
+            plugin(provider(new Callable<RegularFile>(){
+                @Override
+                RegularFile call() throws Exception {
+                    return new RegularFile() {
+                        @Override
+                        File getAsFile() {
+                            if (new File("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion").exists()) {
+                                project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion"))
                             }
+                            project.mkdir bwcJSPluginPath + bwcVersion
+                            ant.get(src: bwcRemoteFileJSPlugin,
+                                    dest: bwcJSPluginPath + bwcVersion,
+                                    httpusecaches: false)
+                            return fileTree(bwcJSPluginPath + bwcVersion).getSingleFile()
                         }
                     }
-                }))
-                plugin(provider(new Callable<RegularFile>() {
-                    @Override
-                    RegularFile call() throws Exception {
-                        return new RegularFile() {
-                            @Override
-                            File getAsFile() {
-                                File dir = new File('./integ-test/' + bwcFilePath + bwcVersion)
-                                if (!dir.exists()) {
-                                    dir.mkdirs()
-                                }
-                                File f = new File(dir, bwcSqlPlugin)
-                                if (!f.exists()) {
-                                    new URL(bwcRemoteFileSqlPlugin).withInputStream{ ins -> f.withOutputStream{ it << ins }}
-                                }
-                                return fileTree(bwcFilePath + bwcVersion).getSingleFile()
+                }
+            }))
+            plugin(provider(new Callable<RegularFile>() {
+                @Override
+                RegularFile call() throws Exception {
+                    return new RegularFile() {
+                        @Override
+                        File getAsFile() {
+                            File dir = new File('./integ-test/' + bwcFilePath + bwcVersion)
+                            if (!dir.exists()) {
+                                dir.mkdirs()
                             }
+                            File f = new File(dir, bwcSqlPlugin)
+                            if (!f.exists()) {
+                                new URL(bwcRemoteFileSqlPlugin).withInputStream{ ins -> f.withOutputStream{ it << ins }}
+                            }
+                            return fileTree(bwcFilePath + bwcVersion).getSingleFile()
                         }
                     }
-                }))
-            }
-            setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-            setting 'http.content_type.required', 'true'
+                }
+            }))
         }
+        setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+        setting 'http.content_type.required', 'true'
     }
 }
-
 List<Provider<RegularFile>> plugins = [
         provider(new Callable<RegularFile>() {
             @Override
@@ -696,34 +693,32 @@ List<Provider<RegularFile>> plugins = [
 ]
 
 
-// Creates 2 test clusters with 3 nodes of the old version.
-2.times { i ->
-    task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
-        useCluster testClusters."${baseName}$i"
-        filter {
-            includeTestsMatching "org.opensearch.sql.bwc.*IT"
-        }
-        systemProperty 'tests.rest.bwcsuite', 'old_cluster'
-        systemProperty 'tests.rest.bwcsuite_round', 'old'
-        systemProperty 'tests.plugin_bwc_version', bwcVersion
-        nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}")
-        nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}")
+// Creates test cluster with 3 nodes of the old version.
+task "${baseName}#oldVersionClusterTask"(type: StandaloneRestIntegTestTask) {
+    useCluster testClusters."${baseName}"
+    filter {
+        includeTestsMatching "org.opensearch.sql.bwc.*IT"
     }
+    systemProperty 'tests.rest.bwcsuite', 'old_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'old'
+    systemProperty 'tests.plugin_bwc_version', bwcVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
 }
 
 // Upgrade one node of the old cluster to new OpenSearch version with upgraded plugin version.
 // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
 // This is also used as a one third upgraded cluster for a rolling upgrade.
 task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
-    useCluster testClusters."${baseName}0"
-    dependsOn "${baseName}#oldVersionClusterTask0"
+    useCluster testClusters."${baseName}"
+    dependsOn "${baseName}#oldVersionClusterTask"
     if (bwcBundleTest){
         doFirst {
-            testClusters."${baseName}0".nextNodeToNextVersion()
+            testClusters."${baseName}".nextNodeToNextVersion()
         }
     } else {
         doFirst {
-            testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+            testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins)
         }
     }
     filter {
@@ -732,23 +727,23 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
     systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
     systemProperty 'tests.rest.bwcsuite_round', 'first'
     systemProperty 'tests.plugin_bwc_version', bwcVersion
-    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
-    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
 }
 
 // Upgrade the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
 // This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
 // This is used for rolling upgrade.
 task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#mixedClusterTask"
-    useCluster testClusters."${baseName}0"
+    useCluster testClusters."${baseName}"
     if (bwcBundleTest){
         doFirst {
-            testClusters."${baseName}0".nextNodeToNextVersion()
+            testClusters."${baseName}".nextNodeToNextVersion()
         }
     } else {
         doFirst {
-            testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+            testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins)
         }
     }
     filter {
@@ -757,23 +752,23 @@ task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTas
     systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
     systemProperty 'tests.rest.bwcsuite_round', 'second'
     systemProperty 'tests.plugin_bwc_version', bwcVersion
-    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
-    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
 }
 
 // Upgrade the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
 // This results in a fully upgraded cluster.
 // This is used for rolling upgrade.
 task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
-    useCluster testClusters."${baseName}0"
+    useCluster testClusters."${baseName}"
     if (bwcBundleTest){
         doFirst {
-            testClusters."${baseName}0".nextNodeToNextVersion()
+            testClusters."${baseName}".nextNodeToNextVersion()
         }
     } else {
         doFirst {
-            testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+            testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins)
         }
     }
     filter {
@@ -783,42 +778,51 @@ task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask)
     systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
     systemProperty 'tests.rest.bwcsuite_round', 'third'
     systemProperty 'tests.plugin_bwc_version', bwcVersion
-    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
-    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
 }
 
 // Upgrade all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
 // at the same time resulting in a fully upgraded cluster.
 task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
-    dependsOn "${baseName}#oldVersionClusterTask1"
-    useCluster testClusters."${baseName}1"
+    dependsOn "${baseName}#oldVersionClusterTask"
+    useCluster testClusters."${baseName}"
     if (bwcBundleTest){
         doFirst {
-            testClusters."${baseName}1".goToNextVersion()
+            testClusters."${baseName}".goToNextVersion()
         }
     } else {
         doFirst {
-            testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
+            testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(plugins)
         }
     }
     filter {
         includeTestsMatching "org.opensearch.sql.bwc.*IT"
     }
     systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
     systemProperty 'tests.plugin_bwc_version', bwcVersion
-    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
-    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
 }
 
-// A bwc test suite which runs all the bwc tasks combined
-task bwcTestSuite(type: StandaloneRestIntegTestTask) {
+// A bwc test suite which runs all the bwc tasks in rolling upgrade
+task bwcTestRollingUpgradeSuite(type: StandaloneRestIntegTestTask) {
     testLogging {
         events "passed", "skipped", "failed"
     }
     exclude '**/*Test*'
     exclude '**/*IT*'
     dependsOn tasks.named("${baseName}#mixedClusterTask")
     dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
+}
+
+// A bwc test suite which runs all the bwc tasks in full restart
+task bwcTestFullRestartSuite(type: StandaloneRestIntegTestTask) {
+    testLogging {
+        events "passed", "skipped", "failed"
+    }
+    exclude '**/*Test*'
+    exclude '**/*IT*'
     dependsOn tasks.named("${baseName}#fullRestartClusterTask")
 }