Skip to content

Commit d8f09d4

Browse files
Chore: TECH-894 kafka bootstrap (#260)
* TECH-894 bootstrapping kafka package * TECH-894 Using rimraf * Update packages/kafka/package.json Co-authored-by: Igor Savin <iselwin@gmail.com> * switching to lokalise biome * Adding node-rdkafka + small tests checking it * Adding kafka test * Trying to fix test * Trying to fix issue by updating node-gyp * Trying to fix CI * Fix vitest warning * Trying to fix issue with threads * Test fix * Improving test * Fixing test * Test simplified * Trying to improve CI * Adding comment * Trying to increase timeout * Avoiding timeout * Increasing timeout even more * Fixing test * Fixing CI --------- Co-authored-by: Igor Savin <iselwin@gmail.com>
1 parent c4808fb commit d8f09d4

12 files changed

Lines changed: 394 additions & 2 deletions

File tree

.github/workflows/ci.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,11 @@ jobs:
2626
run: |
2727
npm install --ignore-scripts
2828
29+
# We need to run node-rdkafka scripts
30+
- name: Run node-rdkafka scripts
31+
working-directory: packages/kafka
32+
run: npm rebuild node-rdkafka
33+
2934
- name: Docker start
3035
run: |
3136
npm run docker:start

docker-compose.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
services:
2+
23
rabbitmq:
34
image: rabbitmq:4.0.4
45
ports:
56
- ${DOCKER_RABBITMQ_CLIENT_PORT:-5672}:5672
67
- ${DOCKER_RABBITMQ_MANAGEMENT_PORT:-15672}:15672
78
volumes:
89
- rabbit_data:/var/lib/rabbitmq
10+
restart: on-failure
911

1012
localstack:
1113
image: localstack/localstack:4.0.2
@@ -24,6 +26,7 @@ services:
2426
volumes:
2527
- '${TMPDIR:-/tmp}/localstack:/var/log/localstack'
2628
- '/var/run/docker.sock:/var/run/docker.sock'
29+
restart: on-failure
2730

2831
redis:
2932
image: redis:6.2.7-alpine
@@ -34,6 +37,13 @@ services:
3437
- redis_data:/redis/data:cached
3538
restart: on-failure
3639

40+
kafka:
41+
image: apache/kafka:latest
42+
container_name: kafka
43+
ports:
44+
- 9092:9092
45+
restart: on-failure
46+
3747
volumes:
3848
rabbit_data:
3949
driver: local

packages/core/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@
2424
"test": "vitest",
2525
"test:coverage": "npm test -- --coverage",
2626
"test:ci": "npm run docker:start:dev && npm run test:coverage && npm run docker:stop:dev",
27-
"docker:start:dev": "",
28-
"docker:stop:dev": "",
27+
"docker:start:dev": "docker compose up -d",
28+
"docker:stop:dev": "docker compose down",
2929
"prepublishOnly": "npm run lint && npm run build"
3030
},
3131
"dependencies": {

packages/kafka/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# @message-queue-toolkit/kafka
2+
3+
In development

packages/kafka/docker-compose.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
services:
2+
3+
kafka:
4+
image: apache/kafka:latest
5+
container_name: kafka
6+
ports:
7+
- 9092:9092
8+
restart: on-failure

packages/kafka/lib/index.ts

Whitespace-only changes.

packages/kafka/lib/test.spec.ts

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import { once } from 'node:events'
2+
import { waitAndRetry } from '@lokalise/universal-ts-utils/node'
3+
import { KafkaConsumer, type Message, Producer, features, librdkafkaVersion } from 'node-rdkafka'
4+
5+
// TODO: to be removed once we have proper tests
6+
describe('Test', () => {
7+
it('should use node-rdkafka', () => {
8+
expect(features).toBeDefined()
9+
expect(librdkafkaVersion).toBeDefined()
10+
})
11+
12+
it('should send and receive a message', { timeout: 10000 }, async () => {
13+
// Given
14+
const brokers = 'localhost:9092'
15+
// Use a fresh, unique topic per run to avoid stale state
16+
const topic = `test-topic-${Date.now()}`
17+
const messageValue = 'My test message'
18+
19+
const receivedMessages: Message[] = []
20+
21+
// Create a producer
22+
const producer = new Producer({
23+
'metadata.broker.list': brokers,
24+
'allow.auto.create.topics': true,
25+
})
26+
producer.connect()
27+
await once(producer, 'ready')
28+
29+
// Create a consumer with a unique group and disable auto-commit for fresh offsets
30+
const consumer = new KafkaConsumer(
31+
{
32+
'group.id': 'test-group',
33+
'metadata.broker.list': brokers,
34+
'allow.auto.create.topics': true,
35+
'enable.auto.commit': false,
36+
},
37+
{ 'auto.offset.reset': 'earliest' },
38+
)
39+
consumer.connect()
40+
41+
await new Promise<void>((resolve, reject) =>
42+
consumer
43+
.on('ready', () => {
44+
consumer.subscribe([topic])
45+
consumer.consume()
46+
resolve()
47+
})
48+
.on('event.error', (err) => reject(err))
49+
.on('data', (data) => {
50+
receivedMessages.push(data)
51+
}),
52+
)
53+
54+
// When
55+
producer.produce(topic, null, Buffer.from(messageValue))
56+
producer.flush()
57+
58+
// Then
59+
await waitAndRetry(() => receivedMessages.length > 0, 10, 800)
60+
expect(receivedMessages).toHaveLength(1)
61+
expect(receivedMessages[0]?.value?.toString()).toBe(messageValue)
62+
63+
// Cleanup
64+
producer.disconnect()
65+
consumer.disconnect()
66+
})
67+
})

packages/kafka/package.json

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
{
2+
"name": "@message-queue-toolkit/kafka",
3+
"version": "0.0.0",
4+
"private": false,
5+
"license": "MIT",
6+
"homepage": "https://github.com/kibertoad/message-queue-toolkit",
7+
"repository": {
8+
"type": "git",
9+
"url": "git://github.com/kibertoad/message-queue-toolkit.git"
10+
},
11+
"description": "Kafka adapter for message-queue-toolkit",
12+
"keywords": [
13+
"message",
14+
"queue",
15+
"queues",
16+
"abstract",
17+
"common",
18+
"utils",
19+
"notification",
20+
"kafka"
21+
],
22+
"files": ["README.md", "LICENSE", "dist"],
23+
"maintainers": [
24+
{
25+
"name": "Igor Savin",
26+
"email": "kibertoad@gmail.com"
27+
}
28+
],
29+
"type": "module",
30+
"main": "./dist/index.js",
31+
"exports": {
32+
".": "./dist/index.js",
33+
"./package.json": "./package.json"
34+
},
35+
"scripts": {
36+
"build": "npm run clean && tsc --project tsconfig.build.json",
37+
"clean": "rimraf dist",
38+
"test": "vitest",
39+
"test:coverage": "npm test -- --coverage",
40+
"test:ci": "npm run docker:start:dev && npm run test:coverage && npm run docker:stop:dev",
41+
"lint": "biome check . && tsc",
42+
"lint:fix": "biome check --write .",
43+
"docker:start:dev": "docker compose up -d",
44+
"docker:stop:dev": "docker compose down",
45+
"prepublishOnly": "npm run lint && npm run build"
46+
},
47+
"dependencies": {
48+
"@lokalise/node-core": "^13.3.0",
49+
"@lokalise/universal-ts-utils": "^4.4.0",
50+
"node-rdkafka": "^3.3.1",
51+
"zod": "^3.23.8"
52+
},
53+
"peerDependencies": {
54+
"@message-queue-toolkit/core": ">=20.0.0",
55+
"@message-queue-toolkit/schemas": ">=2.0.0"
56+
},
57+
"devDependencies": {
58+
"@biomejs/biome": "1.9.4",
59+
"@lokalise/biome-config": "^2.0.0",
60+
"@lokalise/tsconfig": "^1.3.0",
61+
"@message-queue-toolkit/core": "*",
62+
"@message-queue-toolkit/schemas": "*",
63+
"@types/node": "^22.7.5",
64+
"@vitest/coverage-v8": "^3.0.7",
65+
"awilix": "^12.0.1",
66+
"awilix-manager": "^6.0.0",
67+
"rimraf": "^6.0.1",
68+
"typescript": "^5.7.2",
69+
"vitest": "^3.0.7"
70+
}
71+
}

0 commit comments

Comments
 (0)