Phase 5: Kafka Integration Tests & CI Workflows - Implementation

Author: Madhavan

.github/workflows/ci.yaml

@@ -85,3 +85,55 @@ jobs:
           -PtestPulsarImage=$PULSAR_IMAGE \
           -PtestPulsarImageTag=$PULSAR_IMAGE_TAG \
           ${{ matrix.module }}:test
+
+  test-kafka:
+    needs: build
+    name: Test Kafka
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    strategy:
+      fail-fast: false
+      matrix:
+        module: ['agent-c4']
+        jdk: ['11', '17']
+        kafkaImage: ['apache/kafka:4.2.0', 'confluentinc/cp-kafka:7.9.6', 'confluentinc/cp-kafka:8.1.0']
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up JDK ${{ matrix.jdk }}
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ matrix.jdk }}
+          distribution: 'adopt'
+
+      - name: Get project version
+        uses: HardNorth/github-version-generate@v1.4.0
+        with:
+          version-source: file
+          version-file: gradle.properties
+          version-file-extraction-pattern: '(?<=version=).+'
+
+      - name: Cache Docker layers
+        uses: actions/cache@v5
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Test with Gradle (Kafka)
+        env:
+          DSE_REPO_USERNAME: ${{ secrets.DSE_REPO_USERNAME }}
+          DSE_REPO_PASSWORD: ${{ secrets.DSE_REPO_PASSWORD }}
+        run: |
+          set -e
+          PREV_IFS=$IFS
+          IFS=':'
+          read -ra KAFKA_FULL_IMAGE <<< "${{ matrix.kafkaImage }}"
+          IFS=$PREV_IFS
+          KAFKA_IMAGE=${KAFKA_FULL_IMAGE[0]}
+          KAFKA_IMAGE_TAG=${KAFKA_FULL_IMAGE[1]}
+                    
+          ./gradlew -Pdse4 -PdseRepoUsername=$DSE_REPO_USERNAME -PdseRepoPassword=$DSE_REPO_PASSWORD \
+          -PtestKafkaImage=$KAFKA_IMAGE \
+          -PtestKafkaImageTag=$KAFKA_IMAGE_TAG \
+          ${{ matrix.module }}:test

agent-c4/src/test/java/com/datastax/oss/cdc/agent/KafkaSingleNodeC4Tests.java

@@ -0,0 +1,52 @@
+/**
+ * Copyright DataStax, Inc 2021.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.datastax.oss.cdc.agent;
+
+import com.datastax.oss.cdc.AgentTestUtil;
+import com.datastax.oss.cdc.KafkaSingleNodeTests;
+import com.datastax.testcontainers.cassandra.CassandraContainer;
+import lombok.extern.slf4j.Slf4j;
+import org.testcontainers.containers.Network;
+import org.testcontainers.utility.DockerImageName;
+
+import java.util.Optional;
+
+@Slf4j
+public class KafkaSingleNodeC4Tests extends KafkaSingleNodeTests {
+
+    public static final DockerImageName CASSANDRA_IMAGE = DockerImageName.parse(
+            Optional.ofNullable(System.getenv("CASSANDRA_IMAGE"))
+                    .orElse("cassandra:" + System.getProperty("cassandraVersion"))
+    ).asCompatibleSubstituteFor("cassandra");
+
+    public KafkaSingleNodeC4Tests() {
+        super(AgentTestUtil.Version.C4);
+    }
+
+    @Override
+    public CassandraContainer<?> createCassandraContainer(int nodeIndex, String kafkaBootstrapServers, Network testNetwork) {
+        return CassandraContainer.createCassandraContainerWithAgentKafka(
+                CASSANDRA_IMAGE, testNetwork, nodeIndex, "c4", kafkaBootstrapServers);
+    }
+
+    @Override
+    public int getSegmentSize() {
+        return 1024 * 1024;
+    }
+
+}
+
+// Made with Bob