test: converting test suite to TestContainers

.github/workflows/ci.yml

@@ -71,7 +71,8 @@ jobs:
   test:
     strategy:
       fail-fast: false
-      # The test suite doesn't support concurrent runs.
+      # Each row spins up its own Docker container via testcontainers, so
+      # keep them sequential to avoid contention on a single runner.
       max-parallel: 1
       matrix:
         include:
@@ -88,8 +89,52 @@ jobs:
           toolchain: ${{ env.rust_version }}
       - run: sudo apt-get update
       - run: sudo apt-get install -y libcurl4-openssl-dev
-      # - run: sudo apt-get install -qy valgrind  # Valgrind currently disabled in testing
-      - run: ./test_suite.sh
+      # `--features zstd` so the zstd compression round-trip test in
+      # `tests/future_producer.rs` actually links zstd. The default build
+      # passes `--disable-zstd` to librdkafka (see rdkafka-sys/build.rs),
+      # which would make a `compression.type=zstd` producer reject the
+      # config at creation.
+      - run: cargo test --features zstd
         env:
           KAFKA_VERSION: ${{ matrix.kafka-version }}
-          TERM: xterm-256color
+          RUST_LOG: off
+          RUST_BACKTRACE: 1
+
+  # Smoke-test the smol and async-std runtime examples against a real
+  # broker. The integration suite covers the tokio path via testcontainers;
+  # this job catches breakage in the alternative runtimes that
+  # `cargo build --all-targets` would miss.
+  runtime-examples:
+    runs-on: ubuntu-24.04
+    services:
+      kafka:
+        image: apache/kafka:4.0.2
+        ports:
+          - 9092:9092
+        env:
+          KAFKA_NODE_ID: 1
+          KAFKA_PROCESS_ROLES: broker,controller
+          KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
+          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
+          KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+          KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
+          KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+          KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+          KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+        options: >-
+          --health-cmd "/opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --list"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 12
+          --health-start-period 30s
+    steps:
+      - uses: actions/checkout@v4
+      - uses: lukka/get-cmake@latest
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: ${{ env.rust_version }}
+      - run: sudo apt-get update
+      - run: sudo apt-get install -y libcurl4-openssl-dev
+      - run: cargo run --example runtime_smol --no-default-features --features cmake-build -- --topic smol
+      - run: cargo run --example runtime_async_std --no-default-features --features cmake-build -- --topic async-std

CONTRIBUTING.md

@@ -26,27 +26,27 @@ The unit tests can run without a Kafka broker present:
 cargo test --lib
 ```
 
-### Automatic testing
+### Integration tests
 
-rust-rdkafka contains a suite of tests which is automatically executed by travis in
-docker-compose. Given the interaction with C code that rust-rdkafka has to do, tests
-are executed in valgrind to check eventual memory errors and leaks.
-
-To run the full suite using docker-compose:
+The integration tests start their own Kafka broker via
+[testcontainers-rs], so all you need locally is a running Docker daemon
+and the usual Rust toolchain:
 
 ```bash
-./test_suite.sh
+cargo test
 ```
 
-To run locally, instead:
+To pick a specific Kafka version (default `4.0`), set `KAFKA_VERSION`:
 
 ```bash
-KAFKA_HOST="kafka_server:9092" cargo test
+KAFKA_VERSION=3.9 cargo test
 ```
 
-In this case there is a broker expected to be running on `KAFKA_HOST`.
-The broker must be configured with default partition number 3 and topic
-autocreation in order for the tests to succeed.
+For the full walkthrough, including how the shared broker is wired up,
+how to add a new test, the helper cheatsheet, and known quirks, see
+[`tests/README.md`](tests/README.md).
+
+[testcontainers-rs]: https://github.com/testcontainers/testcontainers-rs
 
 ## Releasing