Skip to content

Commit 3a9bb55

Browse files
CoderYellowCoderYellow
committed
feat: enhance Kerberos configuration and container file handling
- Added `realm` and `domain` fields to Kerberos configuration for dynamic generation of principal names. - Extended Kerberos options with new fields: `materialDirectory`, `localKrb5ConfPath`, and `localClientKeytabPath`. - Updated handling of HDFS, Hive Metastore, Kafka, and Kafka-UI Kerberos principals to use dynamically resolved values. - Updated `BigDataContainerFactory` to use `localKerberosPath` for file copying. - Adjusted test resources to use absolute paths for Kerberos-related configurations. - Added `hdfsNameNode` to the test configuration ports section.
1 parent d34cb75 commit 3a9bb55

5 files changed

Lines changed: 32 additions & 9 deletions

File tree

‎core/src/main/kotlin/org/openprojectx/bigdata/test/core/container/BigDataContainerFactory.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1226,7 +1226,7 @@ internal class BigDataContainerFactory(
12261226
source: String,
12271227
destination: String,
12281228
) {
1229-
val path = hostKerberosMaterialPath(destination)
1229+
val path = Path.of(localKerberosPath(destination))
12301230
Files.createDirectories(path.parent)
12311231
container.copyFileFromContainer(source, path.toString())
12321232
}

‎example/spark/src/test/resources/spark-bigdata-test-apache-hms-kerberos.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@ materialTimeoutSeconds = 120
1111
adminAttempts = 120
1212
adminRetryDelaySeconds = 1
1313
debug = true
14-
materialDirectory = "build/bigdata-test/kerberos"
15-
localKrb5ConfPath = "build/bigdata-test/kerberos/krb5.conf"
16-
localClientKeytabPath = "build/bigdata-test/kerberos/app_user.keytab"
14+
materialDirectory = "/data/Git/bigdata-test/example/spark/build/bigdata-test/kerberos"
15+
localKrb5ConfPath = "/data/Git/bigdata-test/example/spark/build/bigdata-test/kerberos/krb5.conf"
16+
localClientKeytabPath = "/data/Git/bigdata-test/example/spark/build/bigdata-test/kerberos/app_user.keytab"
1717

1818
[ports]
1919
kafka = 19092

‎example/spark/src/test/resources/spark-bigdata-test-common.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ fakeGcs = true
1919

2020
[ports]
2121
hdfsDataNode = 9866
22+
hdfsNameNode = 8020
2223

2324
[containerLogs]
2425
mode = "FILE"

‎junit5/src/main/kotlin/org/openprojectx/bigdata/test/junit5/BigDataTestConfig.kt‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,13 @@ internal data class BigDataTestHdfsConfig(
6363
}
6464

6565
internal data class BigDataTestKerberosConfig(
66+
val realm: String? = null,
67+
val domain: String? = null,
6668
val clientPrincipal: String? = null,
6769
val clientPassword: String? = null,
70+
val materialDirectory: String? = null,
71+
val localKrb5ConfPath: String? = null,
72+
val localClientKeytabPath: String? = null,
6873
val startupTimeoutSeconds: Int? = null,
6974
val materialTimeoutSeconds: Int? = null,
7075
val adminAttempts: Int? = null,
@@ -73,8 +78,13 @@ internal data class BigDataTestKerberosConfig(
7378
) {
7479
fun merge(override: BigDataTestKerberosConfig): BigDataTestKerberosConfig =
7580
BigDataTestKerberosConfig(
81+
realm = override.realm ?: realm,
82+
domain = override.domain ?: domain,
7683
clientPrincipal = override.clientPrincipal ?: clientPrincipal,
7784
clientPassword = override.clientPassword ?: clientPassword,
85+
materialDirectory = override.materialDirectory ?: materialDirectory,
86+
localKrb5ConfPath = override.localKrb5ConfPath ?: localKrb5ConfPath,
87+
localClientKeytabPath = override.localClientKeytabPath ?: localClientKeytabPath,
7888
startupTimeoutSeconds = override.startupTimeoutSeconds ?: startupTimeoutSeconds,
7989
materialTimeoutSeconds = override.materialTimeoutSeconds ?: materialTimeoutSeconds,
8090
adminAttempts = override.adminAttempts ?: adminAttempts,
@@ -312,8 +322,13 @@ internal class BigDataTestConfigLoader(
312322
fakeGcs = services.boolean("fakeGcs"),
313323
),
314324
kerberos = BigDataTestKerberosConfig(
325+
realm = kerberos.string("realm"),
326+
domain = kerberos.string("domain"),
315327
clientPrincipal = kerberos.string("clientPrincipal"),
316328
clientPassword = kerberos.string("clientPassword"),
329+
materialDirectory = kerberos.string("materialDirectory"),
330+
localKrb5ConfPath = kerberos.string("localKrb5ConfPath"),
331+
localClientKeytabPath = kerberos.string("localClientKeytabPath"),
317332
startupTimeoutSeconds = kerberos.int("startupTimeoutSeconds"),
318333
materialTimeoutSeconds = kerberos.int("materialTimeoutSeconds"),
319334
adminAttempts = kerberos.int("adminAttempts"),

‎junit5/src/main/kotlin/org/openprojectx/bigdata/test/junit5/BigDataTestExtension.kt‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,9 @@ class BigDataTestExtension : BeforeAllCallback, AfterAllCallback, ParameterResol
119119
val kafkaUiKerberos = annotation.kafkaUiKerberos || services.kafkaUiKerberos == true
120120
val localStackS3 = annotation.localStackS3 || services.localStackS3 == true
121121
val fakeGcs = annotation.fakeGcs || services.fakeGcs == true
122+
val defaultKerberos = KerberosOptions()
123+
val kerberosRealm = kerberosConfig.realm ?: defaultKerberos.realm
124+
val kerberosDomain = kerberosConfig.domain ?: defaultKerberos.domain
122125
val tlsEnabled = tls.enabled == true ||
123126
config.hdfsWebTls.enabled == true ||
124127
hiveMetastoreTls ||
@@ -141,11 +144,12 @@ class BigDataTestExtension : BeforeAllCallback, AfterAllCallback, ParameterResol
141144
}
142145

143146
if (kerberos || hdfsKerberos || hiveMetastoreKerberos || kafkaKerberos || kafkaUiKerberos) {
144-
val defaultKerberos = KerberosOptions()
145147
builder.withKerberos(
146148
KerberosOptions(
147149
enabled = true,
148150
image = images.kerberos ?: "ghcr.io/openprojectx/directory-kerby/kerby-kdc:latest",
151+
realm = kerberosRealm,
152+
domain = kerberosDomain,
149153
clientPrincipal = annotation.kerberosClientPrincipal
150154
.takeIf { it != BigDataTestDefaults.KERBEROS_CLIENT_PRINCIPAL }
151155
?: kerberosConfig.clientPrincipal
@@ -154,6 +158,9 @@ class BigDataTestExtension : BeforeAllCallback, AfterAllCallback, ParameterResol
154158
.takeIf { it != BigDataTestDefaults.KERBEROS_CLIENT_PASSWORD }
155159
?: kerberosConfig.clientPassword
156160
?: defaultKerberos.clientPassword,
161+
materialDirectory = kerberosConfig.materialDirectory,
162+
localKrb5ConfPath = kerberosConfig.localKrb5ConfPath,
163+
localClientKeytabPath = kerberosConfig.localClientKeytabPath,
157164
startupTimeoutSeconds = kerberosConfig.startupTimeoutSeconds ?: defaultKerberos.startupTimeoutSeconds,
158165
materialTimeoutSeconds = kerberosConfig.materialTimeoutSeconds ?: defaultKerberos.materialTimeoutSeconds,
159166
adminAttempts = kerberosConfig.adminAttempts ?: defaultKerberos.adminAttempts,
@@ -173,7 +180,7 @@ class BigDataTestExtension : BeforeAllCallback, AfterAllCallback, ParameterResol
173180
webTls = config.hdfsWebTls.toHttpTls("localhost"),
174181
kerberos = KerberosAuthOptions(
175182
enabled = hdfsKerberos,
176-
servicePrincipal = "nn/hdfs.example.com@EXAMPLE.COM",
183+
servicePrincipal = "nn/hdfs.$kerberosDomain@$kerberosRealm",
177184
keytabPath = "/kerby/keytabs/hdfs-namenode.keytab",
178185
),
179186
),
@@ -202,7 +209,7 @@ class BigDataTestExtension : BeforeAllCallback, AfterAllCallback, ParameterResol
202209
tls = config.hiveMetastoreTls.toHttpTls("localhost").copy(enabled = hiveMetastoreTls),
203210
kerberos = KerberosAuthOptions(
204211
enabled = hiveMetastoreKerberos,
205-
servicePrincipal = "hive/hive-metastore.example.com@EXAMPLE.COM",
212+
servicePrincipal = "hive/hive-metastore.$kerberosDomain@$kerberosRealm",
206213
keytabPath = "/kerby/keytabs/hive-metastore.keytab",
207214
),
208215
),
@@ -222,12 +229,12 @@ class BigDataTestExtension : BeforeAllCallback, AfterAllCallback, ParameterResol
222229
kafkaUiTls = config.kafkaUiTls.toHttpTls("localhost"),
223230
kerberos = KerberosAuthOptions(
224231
enabled = kafkaKerberos,
225-
servicePrincipal = "kafka/localhost@EXAMPLE.COM",
232+
servicePrincipal = "kafka/localhost@$kerberosRealm",
226233
keytabPath = "/kerby/keytabs/kafka-broker1.keytab",
227234
),
228235
kafkaUiKerberos = KerberosAuthOptions(
229236
enabled = kafkaUiKerberos,
230-
servicePrincipal = "kafbat-ui/kafbat-ui.example.com@EXAMPLE.COM",
237+
servicePrincipal = "kafbat-ui/kafbat-ui.$kerberosDomain@$kerberosRealm",
231238
keytabPath = "/kerby/keytabs/kafbat-ui.keytab",
232239
),
233240
),

0 commit comments

Comments
 (0)