From b937757b2c45d6a052c33649e0824310fe56859b Mon Sep 17 00:00:00 2001 From: Joe Bowbeer Date: Wed, 3 Dec 2025 05:53:10 +0000 Subject: [PATCH] confluentinc kafka-javascript Signed-off-by: Joe Bowbeer --- package-lock.json | 252 ++++++++++++++++-- packages/modules/kafka/package.json | 2 +- .../kafka/src/kafka-container-7.test.ts | 46 ++-- .../kafka/src/kafka-container-latest.test.ts | 23 +- packages/modules/kafka/src/test-helper.ts | 18 +- packages/modules/redpanda/package.json | 2 +- packages/modules/redpanda/src/test-helper.ts | 13 +- 7 files changed, 292 insertions(+), 64 deletions(-) diff --git a/package-lock.json b/package-lock.json index 56ebc47fc..a724218ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2032,6 +2032,27 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/@confluentinc/kafka-javascript": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@confluentinc/kafka-javascript/-/kafka-javascript-1.7.0.tgz", + "integrity": "sha512-nmb7+TYjokCFOX2qUgSqhSW7dCy6/raYYneqR/dLDQwUPak4yuRLGfrvGHTvoH0VC4BUUiRu8niJS//5c3SC/A==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "workspaces": [ + ".", + "schemaregistry", + "schemaregistry-examples" + ], + "dependencies": { + "@mapbox/node-pre-gyp": "^1.0.11", + "bindings": "^1.3.1", + "nan": "^2.22.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@couchbase/couchbase-darwin-arm64-napi": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/@couchbase/couchbase-darwin-arm64-napi/-/couchbase-darwin-arm64-napi-4.6.0.tgz", @@ -4577,6 +4598,159 @@ "uuid": "dist/esm/bin/uuid" } }, + "node_modules/@mapbox/node-pre-gyp": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz", + "integrity": "sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "detect-libc": "^2.0.0", + "https-proxy-agent": "^5.0.0", + "make-dir": "^3.1.0", + "node-fetch": "^2.6.7", + "nopt": "^5.0.0", + "npmlog": "^5.0.1", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.11" + }, + "bin": { + "node-pre-gyp": "bin/node-pre-gyp" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "dev": true, + "license": "ISC" + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/are-we-there-yet": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz", + "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==", + "deprecated": "This package is no longer supported.", + "dev": true, + "license": "ISC", + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/gauge": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz", + "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==", + "deprecated": "This package is no longer supported.", + "dev": true, + "license": "ISC", + "dependencies": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.2", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.1", + "object-assign": "^4.1.1", + "signal-exit": "^3.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/make-dir/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/npmlog": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz", + "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==", + "deprecated": "This package is no longer supported.", + "dev": true, + "license": "ISC", + "dependencies": { + "are-we-there-yet": "^2.0.0", + "console-control-strings": "^1.1.0", + "gauge": "^3.0.0", + "set-blocking": "^2.0.0" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/@mongodb-js/saslprep": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.3.2.tgz", @@ -8886,6 +9060,16 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -10727,6 +10911,17 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -11604,6 +11799,13 @@ "node": ">=16.0.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "dev": true, + "license": "MIT" + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -14178,12 +14380,13 @@ } }, "node_modules/jsonwebtoken/node_modules/jws": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", - "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.3.tgz", + "integrity": "sha512-byiJ0FLRdLdSVSReO/U4E7RoEyOCKnEnEPMjq3HxWtvzLsV08/i5RQKsFVNkCldrCaPr2vDNAOMsfs8T/Hze7g==", "dev": true, + "license": "MIT", "dependencies": { - "jwa": "^1.4.1", + "jwa": "^1.4.2", "safe-buffer": "^5.0.1" } }, @@ -14293,24 +14496,16 @@ } }, "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", "dev": true, + "license": "MIT", "dependencies": { - "jwa": "^2.0.0", + "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, - "node_modules/kafkajs": { - "version": "2.2.4", - "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-2.2.4.tgz", - "integrity": "sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==", - "dev": true, - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/kareem": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/kareem/-/kareem-3.0.0.tgz", @@ -15903,10 +16098,11 @@ } }, "node_modules/nan": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", - "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==", - "optional": true + "version": "2.23.1", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.23.1.tgz", + "integrity": "sha512-r7bBUGKzlqk8oPBDYxt6Z0aEdF1G1rwlMcLk8LCOMbOzf0mG+JUfUzG4fIMWwHWP0iyaLWEQZJmtB7nOHEm/qw==", + "devOptional": true, + "license": "MIT" }, "node_modules/nano": { "version": "11.0.3", @@ -16573,6 +16769,16 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/object-hash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", @@ -22420,7 +22626,7 @@ "testcontainers": "^11.9.0" }, "devDependencies": { - "kafkajs": "^2.2.4" + "@confluentinc/kafka-javascript": "^1.7.0" } }, "packages/modules/kurrentdb": { @@ -22614,7 +22820,7 @@ "testcontainers": "^11.9.0" }, "devDependencies": { - "kafkajs": "^2.2.4" + "@confluentinc/kafka-javascript": "^1.7.0" } }, "packages/modules/s3mock": { diff --git a/packages/modules/kafka/package.json b/packages/modules/kafka/package.json index a869a7650..d901632b3 100644 --- a/packages/modules/kafka/package.json +++ b/packages/modules/kafka/package.json @@ -29,7 +29,7 @@ "build": "tsc --project tsconfig.build.json" }, "devDependencies": { - "kafkajs": "^2.2.4" + "@confluentinc/kafka-javascript": "^1.7.0" }, "dependencies": { "compare-versions": "^6.1.1", diff --git a/packages/modules/kafka/src/kafka-container-7.test.ts b/packages/modules/kafka/src/kafka-container-7.test.ts index 44d6da5a6..1026e0431 100644 --- a/packages/modules/kafka/src/kafka-container-7.test.ts +++ b/packages/modules/kafka/src/kafka-container-7.test.ts @@ -84,17 +84,20 @@ describe("KafkaContainer", { timeout: 240_000 }, () => { }) .start(); - await assertMessageProducedAndConsumed(container, { - brokers: [`${container.getHost()}:${container.getMappedPort(9096)}`], - sasl: { - username: "app-user", - password: "userPassword", - mechanism: "scram-sha-512", - }, - ssl: { - ca: [fs.readFileSync(path.resolve(certificatesDir, "kafka.client.truststore.pem"))], + await assertMessageProducedAndConsumed( + container, + { + brokers: [`${container.getHost()}:${container.getMappedPort(9096)}`], + ssl: true, }, - }); + { + "sasl.mechanism": "SCRAM-SHA-512", + "sasl.username": "app-user", + "sasl.password": "userPassword", + "security.protocol": "sasl_ssl", + "ssl.ca.location": path.resolve(certificatesDir, "kafka.client.truststore.pem"), + } + ); // } }); @@ -121,17 +124,20 @@ describe("KafkaContainer", { timeout: 240_000 }, () => { }) .start(); - await assertMessageProducedAndConsumed(container, { - brokers: [`${container.getHost()}:${container.getMappedPort(9096)}`], - sasl: { - username: "app-user", - password: "userPassword", - mechanism: "scram-sha-512", - }, - ssl: { - ca: [fs.readFileSync(path.resolve(certificatesDir, "kafka.client.truststore.pem"))], + await assertMessageProducedAndConsumed( + container, + { + brokers: [`${container.getHost()}:${container.getMappedPort(9096)}`], + ssl: true, }, - }); + { + "sasl.mechanism": "SCRAM-SHA-512", + "sasl.username": "app-user", + "sasl.password": "userPassword", + "security.protocol": "sasl_ssl", + "ssl.ca.location": path.resolve(certificatesDir, "kafka.client.truststore.pem"), + } + ); }); it(`should connect within Docker network`, async () => { diff --git a/packages/modules/kafka/src/kafka-container-latest.test.ts b/packages/modules/kafka/src/kafka-container-latest.test.ts index 3197080ca..888fbdb30 100644 --- a/packages/modules/kafka/src/kafka-container-latest.test.ts +++ b/packages/modules/kafka/src/kafka-container-latest.test.ts @@ -55,17 +55,20 @@ describe("KafkaContainer", { timeout: 240_000 }, () => { await using container = await new KafkaContainer(IMAGE).withSaslSslListener(saslConfig).start(); - await assertMessageProducedAndConsumed(container, { - brokers: [`${container.getHost()}:${container.getMappedPort(9096)}`], - sasl: { - username: "app-user", - password: "userPassword", - mechanism: "scram-sha-512", - }, - ssl: { - ca: [fs.readFileSync(path.resolve(certificatesDir, "kafka.client.truststore.pem"))], + await assertMessageProducedAndConsumed( + container, + { + brokers: [`${container.getHost()}:${container.getMappedPort(9096)}`], + ssl: true, }, - }); + { + "sasl.mechanism": "SCRAM-SHA-512", + "sasl.username": "app-user", + "sasl.password": "userPassword", + "security.protocol": "sasl_ssl", + "ssl.ca.location": path.resolve(certificatesDir, "kafka.client.truststore.pem"), + } + ); // } }); diff --git a/packages/modules/kafka/src/test-helper.ts b/packages/modules/kafka/src/test-helper.ts index 22111a0c5..ffcfbb86b 100644 --- a/packages/modules/kafka/src/test-helper.ts +++ b/packages/modules/kafka/src/test-helper.ts @@ -1,21 +1,29 @@ -import { Kafka, KafkaConfig, logLevel } from "kafkajs"; +import { GlobalConfig, KafkaJS } from "@confluentinc/kafka-javascript"; import { StartedKafkaContainer } from "./kafka-container"; // kafkaTestHelper { export async function assertMessageProducedAndConsumed( container: StartedKafkaContainer, - additionalConfig: Partial = {} + additionalKafkaConfig: Partial = {}, + additionalGlobalConfig: Partial = {} ) { const brokers = [`${container.getHost()}:${container.getMappedPort(9093)}`]; - const kafka = new Kafka({ logLevel: logLevel.NOTHING, brokers: brokers, ...additionalConfig }); + const kafka = new KafkaJS.Kafka({ + kafkaJS: { + logLevel: KafkaJS.logLevel.ERROR, + brokers, + ...additionalKafkaConfig, + }, + ...additionalGlobalConfig, + }); const producer = kafka.producer(); await producer.connect(); - const consumer = kafka.consumer({ groupId: "test-group" }); + const consumer = kafka.consumer({ kafkaJS: { groupId: "test-group", fromBeginning: true } }); await consumer.connect(); await producer.send({ topic: "test-topic", messages: [{ value: "test message" }] }); - await consumer.subscribe({ topic: "test-topic", fromBeginning: true }); + await consumer.subscribe({ topic: "test-topic" }); const consumedMessage = await new Promise((resolve) => consumer.run({ diff --git a/packages/modules/redpanda/package.json b/packages/modules/redpanda/package.json index c04516937..a75c1e26e 100644 --- a/packages/modules/redpanda/package.json +++ b/packages/modules/redpanda/package.json @@ -33,6 +33,6 @@ "testcontainers": "^11.9.0" }, "devDependencies": { - "kafkajs": "^2.2.4" + "@confluentinc/kafka-javascript": "^1.7.0" } } diff --git a/packages/modules/redpanda/src/test-helper.ts b/packages/modules/redpanda/src/test-helper.ts index d156a7a36..1d07fb34a 100644 --- a/packages/modules/redpanda/src/test-helper.ts +++ b/packages/modules/redpanda/src/test-helper.ts @@ -1,17 +1,22 @@ -import { Kafka, logLevel } from "kafkajs"; +import { KafkaJS } from "@confluentinc/kafka-javascript"; import { StartedRedpandaContainer } from "./redpanda-container"; // redpandaTestHelper { export async function assertMessageProducedAndConsumed(container: StartedRedpandaContainer) { - const kafka = new Kafka({ logLevel: logLevel.NOTHING, brokers: [container.getBootstrapServers()] }); + const kafka = new KafkaJS.Kafka({ + kafkaJS: { + logLevel: KafkaJS.logLevel.NOTHING, + brokers: [container.getBootstrapServers()], + }, + }); const producer = kafka.producer(); await producer.connect(); - const consumer = kafka.consumer({ groupId: "test-group" }); + const consumer = kafka.consumer({ kafkaJS: { groupId: "test-group", fromBeginning: true } }); await consumer.connect(); await producer.send({ topic: "test-topic", messages: [{ value: "test message" }] }); - await consumer.subscribe({ topic: "test-topic", fromBeginning: true }); + await consumer.subscribe({ topic: "test-topic" }); const consumedMessage = await new Promise((resolve) => consumer.run({