Merge pull request #2435 from redpanda-data/chore/e2e-test-variant-kafka

e2e tests variant for Kafka

Author: jvorcak

frontend/tests/shared/global-setup.mjs

@@ -1,6 +1,6 @@
 import { GenericContainer, Network, Wait } from 'testcontainers';
 
-import { KAFKA_CONNECT_IMAGE, OWL_SHOP_IMAGE, REDPANDA_IMAGE } from './test-images.mjs';
+import { KAFKA_CONNECT_IMAGE, KAFKA_IMAGE, OWL_SHOP_IMAGE, REDPANDA_IMAGE } from './test-images.mjs';
 import { exec } from 'node:child_process';
 import { existsSync, readFileSync, realpathSync, writeFileSync } from 'node:fs';
 import { dirname, join, resolve } from 'node:path';
@@ -156,6 +156,39 @@ async function verifyRedpandaServices(state, ports) {
   }
 }
 
+async function startKafkaContainer(network, state, ports, variantName) {
+  console.log('Starting Apache Kafka container (KRaft, SASL/PLAIN)...');
+  const jaasConfPath = resolve(__dirname, '..', `test-variant-${variantName}`, 'config', 'kafka_server_jaas.conf');
+  const kafka = await new GenericContainer(KAFKA_IMAGE)
+    .withNetwork(network)
+    .withNetworkAliases('kafka')
+    .withExposedPorts({ container: 9092, host: ports.kafkaKafka })
+    .withBindMounts([{ source: jaasConfPath, target: '/etc/kafka/jaas.conf', mode: 'ro' }])
+    .withEnvironment({
+      KAFKA_NODE_ID: '1',
+      KAFKA_PROCESS_ROLES: 'broker,controller',
+      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:9093',
+      KAFKA_LISTENERS: 'SASL_PLAINTEXT://:9092,CONTROLLER://:9093',
+      KAFKA_ADVERTISED_LISTENERS: 'SASL_PLAINTEXT://kafka:9092',
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'SASL_PLAINTEXT:SASL_PLAINTEXT,CONTROLLER:PLAINTEXT',
+      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER',
+      KAFKA_INTER_BROKER_LISTENER_NAME: 'SASL_PLAINTEXT',
+      KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN',
+      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN',
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1',
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true',
+      CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk',
+      KAFKA_OPTS: '-Djava.security.auth.login.config=/etc/kafka/jaas.conf',
+    })
+    .withWaitStrategy(Wait.forLogMessage(/Kafka Server started/i))
+    .withStartupTimeout(120_000)
+    .start();
+
+  state.kafkaId = kafka.getId();
+  state.kafkaContainer = kafka;
+  console.log(`✓ Apache Kafka container started: ${state.kafkaId}`);
+}
+
 async function startOwlShop(network, state) {
   console.log('Starting OwlShop container...');
   const owlshopConfigContent = `
@@ -942,6 +975,12 @@ async function cleanupOnFailure(state) {
       console.log(`Failed to stop destination Redpanda container: ${error.message}`);
     });
   }
+  if (state.kafkaContainer) {
+    console.log('Stopping Kafka container using testcontainers API...');
+    await state.kafkaContainer.stop().catch((error) => {
+      console.log(`Failed to stop Kafka container: ${error.message}`);
+    });
+  }
   if (state.redpandaContainer) {
     console.log('Stopping Redpanda container using testcontainers API...');
     await state.redpandaContainer.stop().catch((error) => {
@@ -1002,6 +1041,24 @@ export default async function globalSetup(config = {}) {
     // Setup Docker infrastructure
     const network = await setupDockerNetwork(state);
 
+    // --- Kafka variant: simple KRaft broker, no Redpanda/owlshop/connect ---
+    if (variantConfig.isKafka) {
+      await startKafkaContainer(network, state, ports, variantName);
+      const backendConfigPath = resolve(__dirname, '..', `test-variant-${variantName}`, 'config', configFile);
+      await startBackendServerWithConfig(
+        network,
+        isEnterprise,
+        imageTag,
+        state,
+        backendConfigPath,
+        ports.backend,
+        'console-backend'
+      );
+      writeFileSync(getStateFile(variantName), JSON.stringify(state, null, 2));
+      console.log('\n✅ All services ready! Starting tests...\n');
+      return;
+    }
+
     // --- Group 1: Start clusters in parallel ---
     const clusterPromises = [
       startRedpandaContainer(network, state, ports).then(() => verifyRedpandaServices(state, ports)),

frontend/tests/shared/global-teardown.mjs

@@ -77,6 +77,12 @@ export default async function globalTeardown(config = {}) {
       });
     }
 
+    if (state.kafkaId) {
+      console.log('Stopping Kafka container...');
+      await execAsync(`docker stop ${state.kafkaId}`).catch(() => {});
+      await execAsync(`docker rm ${state.kafkaId}`).catch(() => {});
+    }
+
     // Stop source cluster (existing/main redpanda)
     if (state.redpandaId) {
       console.log('Stopping source Redpanda container...');

frontend/tests/shared/test-images.mjs

@@ -43,3 +43,9 @@ export const KAFKA_CONNECT_IMAGE = process.env.TEST_IMAGE_KAFKA_CONNECT || forma
  * Override with TEST_IMAGE_OWL_SHOP environment variable.
  */
 export const OWL_SHOP_IMAGE = process.env.TEST_IMAGE_OWL_SHOP || formatImage(config.images.owlShop);
+
+/**
+ * Apache Kafka (bitnami) Docker image.
+ * Override with TEST_IMAGE_KAFKA environment variable.
+ */
+export const KAFKA_IMAGE = process.env.TEST_IMAGE_KAFKA || formatImage(config.images.kafka);

frontend/tests/test-variant-kafka/config/console.config.yaml

@@ -0,0 +1,20 @@
+serveFrontend: true
+
+kafka:
+  brokers: ["kafka:9092"]
+  sasl:
+    enabled: true
+    mechanism: PLAIN
+    username: e2euser
+    password: very-secret
+
+schemaRegistry:
+  enabled: false
+
+redpanda:
+  adminApi:
+    enabled: false
+
+server:
+  listenPort: 3000
+  allowedOrigins: ["http://localhost:3000", "http://localhost:3001", "http://localhost:3002"]

frontend/tests/test-variant-kafka/config/kafka_server_jaas.conf

@@ -0,0 +1,12 @@
+KafkaServer {
+  org.apache.kafka.common.security.plain.PlainLoginModule required
+  username="e2euser"
+  password="very-secret"
+  user_e2euser="very-secret";
+};
+
+KafkaClient {
+  org.apache.kafka.common.security.plain.PlainLoginModule required
+  username="e2euser"
+  password="very-secret";
+};

frontend/tests/test-variant-kafka/config/variant.json

@@ -0,0 +1,12 @@
+{
+  "name": "kafka",
+  "displayName": "Apache Kafka",
+  "isEnterprise": false,
+  "isKafka": true,
+  "needsShadowlink": false,
+  "needsAuth": false,
+  "ports": {
+    "backend": 3002,
+    "kafkaKafka": 19094
+  }
+}

frontend/tests/test-variant-kafka/playwright.config.ts

@@ -0,0 +1,63 @@
+import { defineConfig, devices } from '@playwright/test';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+const reporters = process.env.CI
+  ? [['github' as const], ['html' as const, { outputFolder: 'playwright-report' }]]
+  : [['markdown' as const], ['html' as const, { outputFolder: 'playwright-report' }]];
+
+const config = defineConfig({
+  expect: {
+    timeout: 60 * 1000,
+  },
+
+  testMatch: '**/*.spec.ts',
+
+  fullyParallel: !!process.env.CI,
+
+  forbidOnly: !!process.env.CI,
+
+  retries: process.env.CI ? 1 : 0,
+
+  workers: process.env.CI ? 4 : undefined,
+
+  reporter: reporters,
+
+  globalSetup: '../shared/global-setup.mjs',
+  globalTeardown: '../shared/global-teardown.mjs',
+
+  metadata: {
+    variant: 'kafka',
+    variantName: 'kafka',
+    configFile: 'console.config.yaml',
+    isEnterprise: false,
+    isKafka: true,
+    needsShadowlink: false,
+  },
+
+  use: {
+    navigationTimeout: 30 * 1000,
+    actionTimeout: 30 * 1000,
+    viewport: { width: 1920, height: 1080 },
+    headless: !!process.env.CI,
+
+    baseURL: process.env.REACT_APP_ORIGIN ?? 'http://localhost:3002',
+
+    trace: 'retain-on-failure',
+    screenshot: 'only-on-failure',
+    video: 'retain-on-failure',
+  },
+
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+        permissions: ['clipboard-read', 'clipboard-write'],
+      },
+    },
+  ],
+});
+
+export default config;

frontend/tests/test-variant-kafka/topics/topics.spec.ts

@@ -0,0 +1,33 @@
+import { expect, test } from '@playwright/test';
+
+import { TopicPage } from '../../test-variant-console/utils/topic-page';
+
+test.describe('Topics - Apache Kafka backend', () => {
+  test('should load the topics list', async ({ page }) => {
+    const topicPage = new TopicPage(page);
+    await topicPage.goToTopicsList();
+
+    await expect(page.getByTestId('topics-table')).toBeVisible();
+  });
+
+  test('should create and delete a topic', async ({ page }) => {
+    const topicPage = new TopicPage(page);
+    const topicName = `kafka-e2e-${Date.now()}`;
+
+    await topicPage.createTopic(topicName);
+    await topicPage.verifyTopicInList(topicName);
+    await topicPage.deleteTopic(topicName);
+    await topicPage.verifyTopicNotInList(topicName);
+  });
+
+  test('should search for a topic by name', async ({ page }) => {
+    const topicPage = new TopicPage(page);
+    const topicName = `kafka-search-${Date.now()}`;
+
+    await topicPage.createTopic(topicName);
+    await topicPage.searchTopics(topicName);
+    await topicPage.verifyTopicInList(topicName);
+
+    await topicPage.deleteTopic(topicName);
+  });
+});

test-images.json

@@ -15,6 +15,10 @@
     "owlShop": {
       "repository": "quay.io/cloudhut/owl-shop",
       "tag": "master"
+    },
+    "kafka": {
+      "repository": "apache/kafka",
+      "tag": "3.9.0"
     }
   }
 }