Skip to content

Commit 688c2bf

Browse files
committed
feat: add compatibility for custom Kafka and PostgreSQL images
- Updated `BigDataContainerFactory` to support Testcontainers image compatibility checks for Kafka and PostgreSQL. - Introduced helper methods to mark overridden Kafka and PostgreSQL images as compatible substitutes. - Enhanced Kerberos configurations to handle escaped characters in JAAS keytab paths and principal values. - Updated user guide with details on Kafka and PostgreSQL image compatibility logic.
1 parent 096dc34 commit 688c2bf

2 files changed

Lines changed: 23 additions & 9 deletions

File tree

core/src/main/kotlin/org/openprojectx/bigdata/test/core/container/BigDataContainerFactory.kt

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ internal class BigDataContainerFactory(
207207
if (hive.distribution == HiveMetastoreDistribution.CLOUDERA) {
208208
return clouderaHms()
209209
}
210-
val postgres = PostgreSQLContainer(DockerImageName.parse(hive.databaseImage))
210+
val postgres = PostgreSQLContainer(compatiblePostgresImage(hive.databaseImage))
211211
.withNetwork(network)
212212
.withNetworkAliases("hive-metastore-postgres")
213213
.withDatabaseName(hive.databaseName)
@@ -257,6 +257,9 @@ internal class BigDataContainerFactory(
257257
}
258258
}
259259

260+
private fun compatiblePostgresImage(image: String): DockerImageName =
261+
DockerImageName.parse(image).asCompatibleSubstituteFor("postgres")
262+
260263
private fun clouderaHms(): BigDataServiceContainer {
261264
val hive = options.hiveMetastore
262265
val container = GenericBigDataContainer(hive.image)
@@ -370,9 +373,9 @@ internal class BigDataContainerFactory(
370373
private fun tlsKafka(kafka: KafkaOptions): BigDataServiceContainer {
371374
val kafkaHostPort = options.portBindings.hostPort(9092, options.portBindings.kafka)
372375
val container = if (kafkaHostPort == 0) {
373-
KafkaContainer(DockerImageName.parse(kafka.image))
376+
KafkaContainer(compatibleKafkaImage(kafka.image))
374377
} else {
375-
FixedPortKafkaContainer(DockerImageName.parse(kafka.image)).withServicePort(9092, kafkaHostPort)
378+
FixedPortKafkaContainer(compatibleKafkaImage(kafka.image)).withServicePort(9092, kafkaHostPort)
376379
}
377380
container
378381
.withNetwork(network)
@@ -402,9 +405,9 @@ internal class BigDataContainerFactory(
402405
private fun plaintextKafka(kafka: KafkaOptions): BigDataServiceContainer {
403406
val kafkaHostPort = options.portBindings.hostPort(9092, options.portBindings.kafka)
404407
val container = if (kafkaHostPort == 0) {
405-
KafkaContainer(DockerImageName.parse(kafka.image))
408+
KafkaContainer(compatibleKafkaImage(kafka.image))
406409
} else {
407-
FixedPortKafkaContainer(DockerImageName.parse(kafka.image)).withServicePort(9092, kafkaHostPort)
410+
FixedPortKafkaContainer(compatibleKafkaImage(kafka.image)).withServicePort(9092, kafkaHostPort)
408411
}
409412
container
410413
.withNetwork(network)
@@ -427,6 +430,9 @@ internal class BigDataContainerFactory(
427430
}
428431
}
429432

433+
private fun compatibleKafkaImage(image: String): DockerImageName =
434+
DockerImageName.parse(image).asCompatibleSubstituteFor("apache/kafka")
435+
430436
private fun schemaRegistry(): BigDataServiceContainer {
431437
val kafka = options.kafka
432438
val container = GenericBigDataContainer(kafka.schemaRegistryImage)
@@ -1206,16 +1212,22 @@ internal class BigDataContainerFactory(
12061212
com.sun.security.auth.module.Krb5LoginModule required
12071213
useKeyTab=true
12081214
storeKey=true
1209-
keyTab="${options.keytabPath}"
1210-
principal="${options.servicePrincipal}";
1215+
keyTab="${jaasKeytab(options.keytabPath)}"
1216+
principal="${jaasValue(options.servicePrincipal)}";
12111217
};
12121218
""".trimIndent()
12131219

12141220
private fun inlineJaas(options: KerberosAuthOptions): String =
1215-
"""com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab="${options.keytabPath}" principal="${options.servicePrincipal}";"""
1221+
"""com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab="${jaasKeytab(options.keytabPath)}" principal="${jaasValue(options.servicePrincipal)}";"""
12161222

12171223
private fun inlineJaas(principal: String, keytabPath: String): String =
1218-
"""com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab="$keytabPath" principal="$principal";"""
1224+
"""com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab="${jaasKeytab(keytabPath)}" principal="${jaasValue(principal)}";"""
1225+
1226+
private fun jaasKeytab(path: String): String =
1227+
jaasValue(path.replace('\\', '/'))
1228+
1229+
private fun jaasValue(value: String): String =
1230+
value.replace("\\", "\\\\").replace("\"", "\\\"")
12191231

12201232
private fun localKerberosPath(containerPath: String): String =
12211233
when {

doc/user-guide.adoc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,8 @@ localStackS3 = "localstack/localstack:4.14.0"
233233
fakeGcs = "fsouza/fake-gcs-server:1.54"
234234
----
235235

236+
Kafka and the open-source HMS PostgreSQL sidecar use typed Testcontainers classes instead of plain generic containers. When you override those images, `bigdata-test` marks the configured Kafka image as a compatible substitute for `apache/kafka` and the configured PostgreSQL image as a compatible substitute for `postgres`, so private mirrors or rebuilt images can still pass Testcontainers image checks.
237+
236238
You can place `[images]` in the same TOML files used by `@BigDataExtensions`, because `@BigDataTest` reads those locations before startup when the extensions annotation is present:
237239

238240
[source,kotlin]

0 commit comments

Comments
 (0)