Skip to content

Commit 33fdd3d

Browse files
CoderYellowCoderYellow
committed
feat: introduce composable Testcontainers-based fixtures and integration for big-data testing
0 parents  commit 33fdd3d

39 files changed

Lines changed: 1939 additions & 0 deletions

‎.gitignore‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
.gradle
2+
build/
3+
!gradle/wrapper/gradle-wrapper.jar
4+
!**/src/main/**/build/
5+
!**/src/test/**/build/
6+
7+
### IntelliJ IDEA ###
8+
.idea/modules.xml
9+
.idea/jarRepositories.xml
10+
.idea/compiler.xml
11+
.idea/libraries/
12+
*.iws
13+
*.iml
14+
*.ipr
15+
out/
16+
!**/src/main/**/out/
17+
!**/src/test/**/out/
18+
19+
### Kotlin ###
20+
.kotlin
21+
22+
### Eclipse ###
23+
.apt_generated
24+
.classpath
25+
.factorypath
26+
.project
27+
.settings
28+
.springBeans
29+
.sts4-cache
30+
bin/
31+
!**/src/main/**/bin/
32+
!**/src/test/**/bin/
33+
34+
### NetBeans ###
35+
/nbproject/private/
36+
/nbbuild/
37+
/dist/
38+
/nbdist/
39+
/.nb-gradle/
40+
41+
### VS Code ###
42+
.vscode/
43+
44+
### Mac OS ###
45+
.DS_Store
46+
/.idea/

‎README.md‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
# bigdata-test
2+
3+
Composable Testcontainers-based fixtures for local big-data integration tests.
4+
5+
## Modules
6+
7+
- `core`: container builder and endpoint/property model
8+
- `junit5`: `@BigDataTest` extension for JUnit 5 tests
9+
- `bigdata-test-spring-boot-autoconfigure`: Spring Boot auto-configuration
10+
- `bigdata-test-spring-boot-starter`: starter that brings in the auto-configuration
11+
- `example:spring`: Spring Boot local-development example
12+
- `example:junit`: JUnit 5 integration-test examples
13+
14+
## Core Usage
15+
16+
```kotlin
17+
val kit = BigDataTestKit.builder()
18+
.withHiveMetastore()
19+
.withKafka(KafkaOptions(enabled = true, schemaRegistryEnabled = true, kafkaUiEnabled = true))
20+
.withLocalStackS3()
21+
.build()
22+
23+
kit.use {
24+
it.start()
25+
val metastoreUri = it.endpoint(BigDataService.HIVE_METASTORE).property("hive.metastore.uris")
26+
val bootstrapServers = it.endpoint(BigDataService.KAFKA).property("bootstrap.servers")
27+
}
28+
```
29+
30+
## JUnit 5
31+
32+
```kotlin
33+
@BigDataTest(hiveMetastore = true, kafka = true, localStackS3 = true)
34+
class MyIntegrationTest {
35+
@Test
36+
fun test(kit: BigDataTestKit) {
37+
val properties = kit.springProperties()
38+
}
39+
}
40+
```
41+
42+
Kerberos can be enabled per service:
43+
44+
```kotlin
45+
@BigDataTest(
46+
kerberos = true,
47+
hiveMetastore = true,
48+
hiveMetastoreKerberos = true,
49+
kafka = true,
50+
kafkaKerberos = true,
51+
schemaRegistry = true,
52+
schemaRegistryKerberos = true,
53+
)
54+
class MyKerberosIntegrationTest
55+
```
56+
57+
## Spring Boot
58+
59+
Add the starter and enable the kit in local-development or test configuration:
60+
61+
```yaml
62+
bigdata:
63+
test:
64+
enabled: true
65+
hive-metastore:
66+
enabled: true
67+
kafka:
68+
enabled: true
69+
kerberos-enabled: true
70+
schema-registry-enabled: true
71+
schema-registry-kerberos-enabled: true
72+
localstack-s3:
73+
enabled: true
74+
```
75+
76+
The auto-configuration exposes a started `BigDataTestKit` bean and closes it with the application context.
77+
78+
## Examples
79+
80+
Run the Spring example without starting containers:
81+
82+
```bash
83+
GRADLE_USER_HOME=/data/.gradle ./gradlew :example:spring:check
84+
```
85+
86+
Run the Spring example with the Kerberos profile when you want it to start containers:
87+
88+
```bash
89+
GRADLE_USER_HOME=/data/.gradle ./gradlew :example:spring:bootRun --args='--spring.profiles.active=kerberos'
90+
```
91+
92+
The JUnit examples in `example/junit` are annotated with `@Disabled`; remove that annotation from an example class to start the configured stack.

‎autoconfigure/bigdata-test-spring-boot-autoconfigure.gradle.kts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
plugins {
2+
id("buildsrc.convention.kotlin-jvm")
3+
`kotlin-kapt`
4+
}
5+
6+
7+
dependencies {
8+
9+
api(project(":core"))
10+
11+
val bootBom = platform("org.springframework.boot:spring-boot-dependencies:${libs.versions.springBoot.get()}")
12+
13+
implementation(bootBom)
14+
kapt(bootBom)
15+
16+
implementation("org.springframework.boot:spring-boot-autoconfigure")
17+
api("org.springframework.boot:spring-boot-starter")
18+
kapt("org.springframework.boot:spring-boot-configuration-processor")
19+
}

‎autoconfigure/src/main/kotlin/org/openprojectx/bigdata/test/autoconfigure/BigDataTestAutoConfiguration.kt‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
package org.openprojectx.bigdata.test.autoconfigure
2+
3+
import org.openprojectx.bigdata.test.core.BigDataTestKit
4+
import org.openprojectx.bigdata.test.core.HdfsOptions
5+
import org.openprojectx.bigdata.test.core.HiveMetastoreOptions
6+
import org.openprojectx.bigdata.test.core.KafkaOptions
7+
import org.openprojectx.bigdata.test.core.KerberosAuthOptions
8+
import org.openprojectx.bigdata.test.core.KerberosOptions
9+
import org.openprojectx.bigdata.test.core.ObjectStoreOptions
10+
import org.openprojectx.bigdata.test.core.TlsOptions
11+
import org.springframework.boot.autoconfigure.AutoConfiguration
12+
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
13+
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
14+
import org.springframework.boot.context.properties.EnableConfigurationProperties
15+
import org.springframework.context.annotation.Bean
16+
17+
@AutoConfiguration
18+
@EnableConfigurationProperties(BigDataTestProperties::class)
19+
@ConditionalOnProperty(prefix = "bigdata.test", name = ["enabled"], havingValue = "true")
20+
class BigDataTestAutoConfiguration {
21+
@Bean(destroyMethod = "close")
22+
@ConditionalOnMissingBean
23+
fun bigDataTestKit(properties: BigDataTestProperties): BigDataTestKit {
24+
val builder = BigDataTestKit.builder()
25+
26+
if (properties.kerberos.enabled) {
27+
builder.withKerberos(
28+
KerberosOptions(
29+
enabled = true,
30+
image = properties.kerberos.image,
31+
realm = properties.kerberos.realm,
32+
domain = properties.kerberos.domain,
33+
clientPrincipal = properties.kerberos.clientPrincipal,
34+
clientPassword = properties.kerberos.clientPassword,
35+
),
36+
)
37+
}
38+
39+
if (properties.tls.enabled) {
40+
builder.withTls(
41+
TlsOptions(
42+
enabled = true,
43+
certPath = properties.tls.certPath,
44+
keyPath = properties.tls.keyPath,
45+
caCertPath = properties.tls.caCertPath,
46+
),
47+
)
48+
}
49+
50+
if (properties.hdfs.enabled) {
51+
builder.withHdfs(
52+
HdfsOptions(
53+
enabled = true,
54+
image = properties.hdfs.image,
55+
kerberos = KerberosAuthOptions(
56+
enabled = properties.hdfs.kerberosEnabled,
57+
servicePrincipal = "nn/hdfs.example.com@${properties.kerberos.realm}",
58+
keytabPath = "/kerby/keytabs/hdfs-namenode.keytab",
59+
),
60+
),
61+
)
62+
}
63+
64+
if (properties.hiveMetastore.enabled) {
65+
builder.withHiveMetastore(
66+
HiveMetastoreOptions(
67+
enabled = true,
68+
image = properties.hiveMetastore.image,
69+
databaseName = properties.hiveMetastore.databaseName,
70+
databaseUser = properties.hiveMetastore.databaseUser,
71+
databasePassword = properties.hiveMetastore.databasePassword,
72+
warehouseDir = properties.hiveMetastore.warehouseDir,
73+
extraConfiguration = properties.hiveMetastore.extraConfiguration,
74+
kerberos = KerberosAuthOptions(
75+
enabled = properties.hiveMetastore.kerberosEnabled,
76+
servicePrincipal = "hive/hive-metastore.example.com@${properties.kerberos.realm}",
77+
keytabPath = "/kerby/keytabs/hive-metastore.keytab",
78+
),
79+
),
80+
)
81+
}
82+
83+
if (properties.kafka.enabled) {
84+
builder.withKafka(
85+
KafkaOptions(
86+
enabled = true,
87+
image = properties.kafka.image,
88+
schemaRegistryEnabled = properties.kafka.schemaRegistryEnabled,
89+
schemaRegistryImage = properties.kafka.schemaRegistryImage,
90+
kafkaUiEnabled = properties.kafka.kafkaUiEnabled,
91+
kafkaUiImage = properties.kafka.kafkaUiImage,
92+
kerberos = KerberosAuthOptions(
93+
enabled = properties.kafka.kerberosEnabled,
94+
servicePrincipal = "kafka/broker1.example.com@${properties.kerberos.realm}",
95+
keytabPath = "/kerby/keytabs/kafka-broker1.keytab",
96+
),
97+
schemaRegistryKerberos = KerberosAuthOptions(
98+
enabled = properties.kafka.schemaRegistryKerberosEnabled,
99+
servicePrincipal = "schema-registry/schema-registry.example.com@${properties.kerberos.realm}",
100+
keytabPath = "/kerby/keytabs/schema-registry.keytab",
101+
),
102+
kafkaUiKerberos = KerberosAuthOptions(
103+
enabled = properties.kafka.kafkaUiKerberosEnabled,
104+
servicePrincipal = "kafbat-ui/kafbat-ui.example.com@${properties.kerberos.realm}",
105+
keytabPath = "/kerby/keytabs/kafbat-ui.keytab",
106+
),
107+
),
108+
)
109+
}
110+
111+
if (properties.localstackS3.enabled) {
112+
builder.withLocalStackS3(ObjectStoreOptions(enabled = true, image = properties.localstackS3.image))
113+
}
114+
115+
if (properties.fakeGcs.enabled) {
116+
builder.withFakeGcs(ObjectStoreOptions(enabled = true, image = properties.fakeGcs.image))
117+
}
118+
119+
return builder.build().also { it.start() }
120+
}
121+
}

‎autoconfigure/src/main/kotlin/org/openprojectx/bigdata/test/autoconfigure/BigDataTestProperties.kt‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package org.openprojectx.bigdata.test.autoconfigure
2+
3+
import org.springframework.boot.context.properties.ConfigurationProperties
4+
5+
@ConfigurationProperties("bigdata.test")
6+
data class BigDataTestProperties(
7+
var enabled: Boolean = false,
8+
var kerberos: Kerberos = Kerberos(),
9+
var tls: Tls = Tls(),
10+
var hdfs: Hdfs = Hdfs(),
11+
var hiveMetastore: HiveMetastore = HiveMetastore(),
12+
var kafka: Kafka = Kafka(),
13+
var localstackS3: ObjectStore = ObjectStore(image = "localstack/localstack:4.14.0"),
14+
var fakeGcs: ObjectStore = ObjectStore(image = "fsouza/fake-gcs-server:1.54"),
15+
) {
16+
data class Kerberos(
17+
var enabled: Boolean = false,
18+
var image: String = "openprojectx/kerby-kdc:latest",
19+
var realm: String = "EXAMPLE.COM",
20+
var domain: String = "example.com",
21+
var clientPrincipal: String = "app_user@EXAMPLE.COM",
22+
var clientPassword: String = "app-user-secret",
23+
)
24+
25+
data class Tls(
26+
var enabled: Boolean = false,
27+
var certPath: String? = null,
28+
var keyPath: String? = null,
29+
var caCertPath: String? = null,
30+
)
31+
32+
data class Hdfs(
33+
var enabled: Boolean = false,
34+
var image: String = "apache/hadoop:3.5.0",
35+
var kerberosEnabled: Boolean = false,
36+
)
37+
38+
data class HiveMetastore(
39+
var enabled: Boolean = false,
40+
var image: String = "ghcr.io/openprojectx/cloudera-hms:0.1.15",
41+
var databaseName: String = "metastore_db",
42+
var databaseUser: String = "hive",
43+
var databasePassword: String = "hive-password",
44+
var warehouseDir: String = "/user/hive/warehouse",
45+
var extraConfiguration: Map<String, String> = emptyMap(),
46+
var kerberosEnabled: Boolean = false,
47+
)
48+
49+
data class Kafka(
50+
var enabled: Boolean = false,
51+
var image: String = "apache/kafka:4.1.2",
52+
var schemaRegistryEnabled: Boolean = false,
53+
var schemaRegistryImage: String = "confluentinc/cp-schema-registry:7.8.0",
54+
var kafkaUiEnabled: Boolean = false,
55+
var kafkaUiImage: String = "ghcr.io/kafbat/kafka-ui:latest",
56+
var kerberosEnabled: Boolean = false,
57+
var schemaRegistryKerberosEnabled: Boolean = false,
58+
var kafkaUiKerberosEnabled: Boolean = false,
59+
)
60+
61+
data class ObjectStore(
62+
var enabled: Boolean = false,
63+
var image: String,
64+
)
65+
}

‎autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
org.openprojectx.bigdata.test.autoconfigure.BigDataTestAutoConfiguration

0 commit comments

Comments
 (0)