Separate MySQL and MariaDB tests (#4517)

Changes:
* Incorporate #4516 making Maria a separate DB in the matrix
* Spawn out the maria-roots of e2e suites
* Introduce TestIntegraion.../mysql and .../mariadb naming pattern for
integration tests, filter from tilt with a matcher. MySQL-only are
TestMySQLOnlyIntegration... to avoid running them from maria targets as
parent tests are first ran to discover subtests. Unit tests are out of
scope for tilt as they don't need any infra

Also GitHub seems to not handle indentation diffs very well, most of the
line changes are just subtest indentation

---------

Co-authored-by: Pablo Francisco Pérez Hidalgo <pablo.perez@clickhouse.com>

Author: ilidemi

.env.example

@@ -14,11 +14,15 @@ CI_CLICKHOUSE_NATIVE_PORT=11000
 CI_MYSQL_HOST=host.docker.internal
 CI_MYSQL_GTID_PORT=3306
 CI_MYSQL_POS_PORT=3307
-CI_MARIADB_PORT=3308
 CI_MYSQL_ROOT_PASSWORD=cipass
 CI_MYSQL_GTID_VERSION=mysql-gtid
 CI_MYSQL_POS_VERSION=mysql-pos
-CI_MARIADB_VERSION=maria-gtid
+CI_SSH_MYSQL_HOST=host.docker.internal
+
+CI_MARIADB_HOST=host.docker.internal
+CI_MARIADB_PORT=3308
+CI_MARIADB_ROOT_PASSWORD=cipass
+CI_MARIADB_VERSION=maria-11
 
 CI_MYSQL_VERSION=mysql-gtid
 CI_MYSQL_PORT=3306 # Default is gtid

.github/workflows/flow.yml

@@ -34,14 +34,13 @@ jobs:
       matrix:
         runner: [ubuntu-latest-16-cores]
         db-version: [
-          {pg: 16, mysql: 'mysql-gtid', mongo: '6.0', ch: 'lts'},
-          {pg: 17, mysql: 'mysql-pos', mongo: '7.0', ch: 'stable'},
-          {pg: 18, mysql: 'maria-gtid', mongo: '8.0', ch: 'latest'},
+          {pg: 16, mysql: 'mysql-gtid', mariadb: 'maria-11', mongo: '6.0', ch: 'lts'},
+          {pg: 17, mysql: 'mysql-pos', mariadb: 'maria-12', mongo: '7.0', ch: 'stable'},
+          {pg: 18, mysql: 'mysql-gtid', mariadb: 'maria-13', mongo: '8.0', ch: 'latest'},
         ]
-        # Per-version container settings consumed by the "MySQL" step, keyed by the matrix
-        # db-version.mysql value. Grouped under "mysql" since these are MySQL/MariaDB families;
-        # only the databases handled by the MySQL step are listed here. Wrapped in a single-item
-        # list because matrix values must be arrays; it stays a single shared value (no extra jobs).
+        # Per-version container settings consumed by the MySQL and MariaDB steps.
+        # Wrapped in a single-item list because matrix values must be arrays; it stays
+        # a single shared value (no extra jobs).
         version-configs:
           - mysql:
               mysql-gtid:
@@ -52,12 +51,17 @@ jobs:
                 img: 'mysql:5.7'
                 env: ['MYSQL_ROOT_PASSWORD=cipass']
                 parameters: ['--log_bin=mysql-bin', '--server-id=1', '--bind-address=::']
-              maria-pos:
-                img: 'mariadb:lts-ubi9@sha256:55a81b2d791d2ff8ad33fef413d9e45e0ac57a951127e0cfc69a8e59f922ba6e'
+            mariadb:
+              maria-11:
+                img: 'mariadb:lts-ubi9'
                 env: ['MARIADB_ROOT_PASSWORD=cipass']
-                parameters: ['--log-bin=maria']
-              maria-gtid:
-                img: 'mariadb:lts-ubi9@sha256:55a81b2d791d2ff8ad33fef413d9e45e0ac57a951127e0cfc69a8e59f922ba6e'
+                parameters: ['--log-bin=maria', '--gtid-strict-mode=ON']
+              maria-12:
+                img: 'mariadb:12-ubi'
+                env: ['MARIADB_ROOT_PASSWORD=cipass']
+                parameters: ['--log-bin=maria', '--gtid-strict-mode=ON']
+              maria-13:
+                img: 'mariadb:13.0-ubi-rc'
                 env: ['MARIADB_ROOT_PASSWORD=cipass']
                 parameters: ['--log-bin=maria', '--gtid-strict-mode=ON']
     runs-on: ${{ matrix.runner }}
@@ -221,6 +225,15 @@ jobs:
           docker run -d --rm --name mysql --network ${{ job.container.network }} -p 3306:3306 \
             -e $DB_ENV $DB_IMG $DB_PARAMS
 
+      - name: MariaDB
+        env:
+          DB_IMG: ${{ matrix.version-configs.mariadb[matrix.db-version.mariadb].img }}
+          DB_ENV: ${{ join(matrix.version-configs.mariadb[matrix.db-version.mariadb].env, ' -e ') }}
+          DB_PARAMS: ${{ join(matrix.version-configs.mariadb[matrix.db-version.mariadb].parameters, ' ') }}
+        run: |
+          docker run -d --rm --name mariadb --network ${{ job.container.network }} -p 3316:3306 \
+            -e $DB_ENV $DB_IMG $DB_PARAMS
+
       - name: Mongo
         run: |
           echo "starting mongoDB..."
@@ -630,7 +643,15 @@ jobs:
           PEERDB_QUEUE_FORCE_TOPIC_CREATION: "true"
           ELASTICSEARCH_TEST_ADDRESS: http://localhost:9200
           CI_PG_VERSION: ${{ matrix.db-version.pg }}
+          CI_MYSQL_HOST: localhost
+          CI_MYSQL_PORT: 3306
+          CI_MYSQL_ROOT_PASSWORD: cipass
           CI_MYSQL_VERSION: ${{ matrix.db-version.mysql }}
+          CI_SSH_MYSQL_HOST: mysql
+          CI_MARIADB_HOST: localhost
+          CI_MARIADB_PORT: 3316
+          CI_MARIADB_ROOT_PASSWORD: cipass
+          CI_MARIADB_VERSION: ${{ matrix.db-version.mariadb }}
           CI_MONGO_ADMIN_URI: mongodb://localhost:27017
           CI_MONGO_ADMIN_USERNAME: "admin"
           CI_MONGO_ADMIN_PASSWORD: "admin"
@@ -669,7 +690,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: peer-flow-logs-pg${{ matrix.db-version.pg }}-my${{ matrix.db-version.mysql }}-mo${{ matrix.db-version.mongo }}
+          name: peer-flow-logs-pg${{ matrix.db-version.pg }}-my${{ matrix.db-version.mysql }}-ma${{ matrix.db-version.mariadb }}-mo${{ matrix.db-version.mongo }}
           path: logs/
           retention-days: 30
 
@@ -688,7 +709,7 @@ jobs:
         if: success() || failure()
         uses: ./.github/actions/ingest-test-results
         with:
-          combination-id: "pg${{ matrix.db-version.pg }}-my${{ matrix.db-version.mysql }}-mo${{ matrix.db-version.mongo }}-ch${{ matrix.db-version.ch }}"
+          combination-id: "pg${{ matrix.db-version.pg }}-my${{ matrix.db-version.mysql }}-ma${{ matrix.db-version.mariadb }}-mo${{ matrix.db-version.mongo }}-ch${{ matrix.db-version.ch }}"
           o11y-api-key-id: ${{ secrets.CI_O11Y_TARGET_API_KEY_ID }}
           o11y-api-key-secret: ${{ secrets.CI_O11Y_TARGET_API_KEY_SECRET }}
           o11y-query-endpoint: ${{ secrets.CI_O11Y_TARGET_QUERY_ENDPOINT }}

README.md

@@ -146,7 +146,7 @@ Available test launchers:
 - **e2e_postgres** -- Postgres to ClickHouse generic tests (`TestGenericCH_PG`)
 - **e2e_mysql-gtid** -- MySQL GTID to ClickHouse generic tests (`TestGenericCH_MySQL`)
 - **e2e_mysql-pos** -- MySQL File-Pos to ClickHouse generic tests (`TestGenericCH_MySQL`)
-- **e2e_mariadb** -- MariaDB to ClickHouse generic tests (`TestGenericCH_MySQL`)
+- **e2e_mariadb** -- MariaDB to ClickHouse generic tests (`TestGenericCH_MariaDB`)
 - **e2e_mongodb** -- MongoDB to ClickHouse test suite (`TestMongoClickhouseSuite`)
 
 Each launcher automatically depends on the required services and provisioning steps, so Tilt will ensure all prerequisites are running before executing the tests. To trigger a test, click the resource in the Tilt UI and press the trigger (play) button.

Tiltfile

@@ -295,11 +295,12 @@ def e2e_test(name, test_run, extra_deps=[], vars_overrides={}):
         allow_parallel=True,
     )
 
-def connector_test(connector, extra_deps=[], vars_overrides={}, name=''):
+def connector_test(connector, extra_deps=[], vars_overrides={}, name='', test_run=''):
     overrides_str = ' '.join(['%s=%s' % (var, value) for var, value in vars_overrides.items()])
+    test_run_arg = (' -run %s' % test_run) if test_run else ''
     local_resource(
         'connector_' + (name or connector),
-        cmd='cd flow && %s go test -count=1 -v ./connectors/%s/...' % (overrides_str, connector),
+        cmd='cd flow && %s go test -count=1 -v%s ./connectors/%s/...' % (overrides_str, test_run_arg, connector),
         labels=['Test'],
         auto_init=False,
         resource_deps=['catalog'] + extra_deps,
@@ -310,14 +311,19 @@ def connector_test(connector, extra_deps=[], vars_overrides={}, name=''):
 mysql_gtid_vars = {
     'CI_MYSQL_PORT': resolve_env('CI_MYSQL_GTID_PORT'),
     'CI_MYSQL_VERSION': resolve_env('CI_MYSQL_GTID_VERSION'),
+    'CI_MYSQL_ROOT_PASSWORD': resolve_env('CI_MYSQL_ROOT_PASSWORD'),
+    'CI_SSH_MYSQL_HOST': resolve_env('CI_SSH_MYSQL_HOST'),
 }
 mysql_pos_vars = {
     'CI_MYSQL_PORT': resolve_env('CI_MYSQL_POS_PORT'),
     'CI_MYSQL_VERSION': resolve_env('CI_MYSQL_POS_VERSION'),
+    'CI_MYSQL_ROOT_PASSWORD': resolve_env('CI_MYSQL_ROOT_PASSWORD'),
+    'CI_SSH_MYSQL_HOST': resolve_env('CI_SSH_MYSQL_HOST'),
 }
 mariadb_vars = {
-    'CI_MYSQL_PORT': resolve_env('CI_MARIADB_PORT'),
-    'CI_MYSQL_VERSION': resolve_env('CI_MARIADB_VERSION'),
+    'CI_MARIADB_PORT': resolve_env('CI_MARIADB_PORT'),
+    'CI_MARIADB_VERSION': resolve_env('CI_MARIADB_VERSION'),
+    'CI_MARIADB_ROOT_PASSWORD': resolve_env('CI_MARIADB_ROOT_PASSWORD'),
 }
 
 # Generic e2e tests
@@ -333,7 +339,7 @@ e2e_test('mysql-gtid', 'TestGenericCH_MySQL', ['provision-mysql-gtid'], vars_ove
 e2e_test('mysql-pos', 'TestGenericCH_MySQL', ['provision-mysql-pos'], vars_overrides=mysql_pos_vars)
 
 # MariaDB to ClickHouse generic tests
-e2e_test('mariadb', 'TestGenericCH_MySQL', ['provision-mariadb'], vars_overrides=mariadb_vars)
+e2e_test('mariadb', 'TestGenericCH_MariaDB', ['provision-mariadb'], vars_overrides=mariadb_vars)
 
 # MongoDB to ClickHouse test suite
 e2e_test('mongodb', 'TestMongoClickhouseSuite', ['provision-mongodb'])
@@ -344,7 +350,7 @@ e2e_test('switchboard-postgres', 'TestSwitchboardPostgres', ['provision-postgres
 
 e2e_test('switchboard-mysql-gtid', 'TestSwitchboardMySQL', ['provision-mysql-gtid'], vars_overrides=mysql_gtid_vars)
 e2e_test('switchboard-mysql-pos', 'TestSwitchboardMySQL', ['provision-mysql-pos'], vars_overrides=mysql_pos_vars)
-e2e_test('switchboard-mariadb', 'TestSwitchboardMySQL', ['provision-mariadb'], vars_overrides=mariadb_vars)
+e2e_test('switchboard-mariadb', 'TestSwitchboardMariaDB', ['provision-mariadb'], vars_overrides=mariadb_vars)
 
 e2e_test('switchboard-mongodb', 'TestSwitchboardMongo', ['provision-mongodb'])
 
@@ -354,25 +360,25 @@ e2e_test('peer-flow-postgres', '^TestPeerFlowE2ETestSuitePG_CH$', ['provision-po
 
 e2e_test('peer-flow-mysql-gtid', '^TestPeerFlowE2ETestSuiteMySQL_CH$', ['provision-mysql-gtid'], vars_overrides=mysql_gtid_vars)
 e2e_test('peer-flow-mysql-pos', '^TestPeerFlowE2ETestSuiteMySQL_CH$', ['provision-mysql-pos'], vars_overrides=mysql_pos_vars)
-e2e_test('peer-flow-mariadb', '^TestPeerFlowE2ETestSuiteMySQL_CH$', ['provision-mariadb'], vars_overrides=mariadb_vars)
+e2e_test('peer-flow-mariadb', '^TestPeerFlowE2ETestSuiteMariaDB_CH$', ['provision-mariadb'], vars_overrides=mariadb_vars)
 
 # API e2e tests
 
 e2e_test('api-postgres', 'TestApiPg', ['provision-postgres'])
 
 e2e_test('api-mysql-gtid', 'TestApiMy', ['provision-mysql-gtid', 'provision-postgres'], vars_overrides=mysql_gtid_vars)
 e2e_test('api-mysql-pos', 'TestApiMy', ['provision-mysql-pos', 'provision-postgres'], vars_overrides=mysql_pos_vars)
-e2e_test('api-mariadb', 'TestApiMy', ['provision-mariadb', 'provision-postgres'], vars_overrides=mariadb_vars)
+e2e_test('api-mariadb', 'TestApiMariaDB', ['provision-mariadb', 'provision-postgres'], vars_overrides=mariadb_vars)
 
 e2e_test('api-mongodb', 'TestApiMongo', ['provision-mongodb'])
 
 # Connectors tests
 
 connector_test('postgres', ['provision-postgres'])
 
-connector_test('mysql', ['provision-mysql-gtid'], vars_overrides=mysql_gtid_vars, name='mysql-gtid')
-connector_test('mysql', ['provision-mysql-pos'], vars_overrides=mysql_pos_vars, name='mysql-pos')
-connector_test('mysql', ['provision-mariadb'], vars_overrides=mariadb_vars, name='mariadb')
+connector_test('mysql', ['provision-mysql-gtid'], vars_overrides=mysql_gtid_vars, name='mysql-gtid', test_run="'(TestMySQLOnlyIntegration|TestIntegration.*)/mysql$'")
+connector_test('mysql', ['provision-mysql-pos'], vars_overrides=mysql_pos_vars, name='mysql-pos', test_run="'(TestMySQLOnlyIntegration|TestIntegration.*)/mysql$'")
+connector_test('mysql', ['provision-mariadb'], vars_overrides=mariadb_vars, name='mariadb', test_run="'TestIntegration.*/mariadb$'")
 
 connector_test('mongo', ['provision-mongodb'])
 

ancillary-docker-compose.yml

@@ -111,11 +111,11 @@ services:
     ports:
       - "${CI_MARIADB_PORT}:3306"
     environment:
-      MARIADB_ROOT_PASSWORD: ${CI_MYSQL_ROOT_PASSWORD}
+      MARIADB_ROOT_PASSWORD: ${CI_MARIADB_ROOT_PASSWORD}
     extra_hosts:
       - "host.docker.internal:host-gateway"
     healthcheck:
-      test: ["CMD", "mariadb-admin", "ping", "-h", "localhost", "-uroot", "-p${CI_MYSQL_ROOT_PASSWORD}"]
+      test: ["CMD", "mariadb-admin", "ping", "-h", "localhost", "-uroot", "-p${CI_MARIADB_ROOT_PASSWORD}"]
       interval: 2s
       timeout: 10s
       retries: 5