Skip to content

Commit 9a25520

Browse files
YokiTokiYokiToki
authored
Merge pull request #27 from Boobl1k/task/WD-696
WD-696 change MinIO to RustFS as target S3, update deserialization and tests
2 parents 6f32b41 + 290c2cf commit 9a25520

10 files changed

Lines changed: 88 additions & 29 deletions

File tree

‎.env‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
1+
STORAGE_PROVIDER=rustfs
2+
# or
3+
# STORAGE_PROVIDER=minio
4+
COMPOSE_PROFILES=${STORAGE_PROVIDER}
15
APP_HOST=127.0.0.30
26
APP_PORT=8080
37
S3_ENDPOINT=http://127.0.0.30:9000
48
S3_REGION=us-east-1
59
S3_BUCKET=test-bucket
6-
MINIO_ACCESS_KEY=my_access_key
7-
MINIO_SECRET_KEY=my_secret_key
10+
S3_ACCESS_KEY=my_access_key
11+
S3_SECRET_KEY=my_secret_key
812
JPG_TEST_OBJECT=100.jpg
913
PNG_TEST_OBJECT=100.png
1014
GIF_TEST_OBJECT=100.gif

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,5 @@ build
22
/.gradle
33
/.idea
44
/data
5+
/minio-data
6+
/rustfs-data

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ repositories {
1010
}
1111

1212
// Append dependency
13-
implementation("com.icerockdev:storage-service:0.10.0")
13+
implementation("com.icerockdev:storage-service:0.11.0")
1414
````
1515

1616
## Library usage

‎docker-compose.yaml‎

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,47 @@
11
version: '3'
22
services:
3-
minio:
4-
image: minio/minio
3+
rustfs:
4+
image: rustfs/rustfs
55
restart: always
6-
logging:
6+
logging: &logging
77
driver: "json-file"
88
options:
99
max-size: "20m"
1010
max-file: "5"
11+
ports:
12+
- "${APP_HOST}:9000:9000"
13+
- "${APP_HOST}:9001:9001"
14+
environment:
15+
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
16+
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
17+
volumes:
18+
- ./rustfs-data:/data
19+
profiles:
20+
- rustfs
21+
rustfs-volume-permission-helper:
22+
image: alpine
23+
volumes:
24+
- ./rustfs-data:/data
25+
command: >
26+
sh -c "
27+
chown -R 10001:10001 /data &&
28+
echo 'Volume Permissions fixed' &&
29+
exit 0
30+
"
31+
restart: "no"
32+
profiles:
33+
- rustfs
34+
minio:
35+
image: minio/minio
36+
restart: always
37+
logging: *logging
1138
environment:
12-
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
13-
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
39+
MINIO_ACCESS_KEY: ${S3_ACCESS_KEY}
40+
MINIO_SECRET_KEY: ${S3_SECRET_KEY}
1441
ports:
15-
- ${APP_HOST}:9000:9000
42+
- "${APP_HOST}:9000:9000"
1643
volumes:
17-
- ./data:/data
44+
- ./minio-data:/data
1845
command: 'server /data'
46+
profiles:
47+
- minio

‎sample/src/main/kotlin/com/icerockdev/sample/Main.kt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
package com.icerockdev.sample
66

77
import com.icerockdev.service.storage.s3.S3StorageImpl
8-
import com.icerockdev.service.storage.s3.minioConfBuilder
8+
import com.icerockdev.service.storage.s3.s3Configuration
99
import io.github.cdimascio.dotenv.dotenv
1010
import io.ktor.http.ContentType
1111
import io.ktor.http.content.PartData
@@ -31,11 +31,11 @@ import software.amazon.awssdk.services.s3.presigner.S3Presigner
3131
object Main {
3232
private val dotenv = dotenv()
3333
private val s3 = S3Client.builder()
34-
.serviceConfiguration(minioConfBuilder)
34+
.serviceConfiguration(s3Configuration)
3535
.credentialsProvider(
3636
StaticCredentialsProvider.create(
3737
AwsBasicCredentials.create(
38-
dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
38+
dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"]
3939
)
4040
)
4141
)
@@ -44,11 +44,11 @@ object Main {
4444
.build()
4545

4646
private val preSigner = S3Presigner.builder()
47-
.serviceConfiguration(minioConfBuilder)
47+
.serviceConfiguration(s3Configuration)
4848
.credentialsProvider(
4949
StaticCredentialsProvider.create(
5050
AwsBasicCredentials.create(
51-
dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
51+
dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"]
5252
)
5353
)
5454
)

‎storage-service/build.gradle.kts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ apply(plugin = "java")
1717
apply(plugin = "kotlin")
1818

1919
group = "com.icerockdev"
20-
version = "0.10.0"
20+
version = "0.11.0"
2121

2222
val sourcesJar by tasks.registering(Jar::class) {
2323
archiveClassifier.set("sources")

‎storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ interface IS3Storage {
7777
fun buildResource(configure: ResourceBuilder.() -> Unit): String
7878
}
7979

80-
val minioConfBuilder: S3Configuration =
80+
val s3Configuration: S3Configuration =
8181
S3Configuration
8282
.builder()
8383
.pathStyleAccessEnabled(true)

‎storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/policy/dto/Principal.kt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,17 @@ package com.icerockdev.service.storage.s3.policy.dto
22

33
import com.fasterxml.jackson.annotation.JsonInclude
44
import com.fasterxml.jackson.annotation.JsonProperty
5+
import com.fasterxml.jackson.core.JsonParser
6+
import com.fasterxml.jackson.core.JsonProcessingException
7+
import com.fasterxml.jackson.databind.DeserializationContext
8+
import com.fasterxml.jackson.databind.JsonDeserializer
9+
import com.fasterxml.jackson.databind.JsonNode
10+
import com.fasterxml.jackson.databind.annotation.JsonDeserialize
511

612
@JsonInclude(JsonInclude.Include.NON_EMPTY)
713
data class Principal(
814
@JsonProperty("AWS")
15+
@JsonDeserialize(using = PrincipalAwsDeserializer::class)
916
val aws: List<String>,
1017
@JsonProperty("CanonicalUser")
1118
val canonicalUser: String?,
@@ -14,3 +21,20 @@ data class Principal(
1421
@JsonProperty("Service")
1522
val service: List<String> = emptyList(),
1623
)
24+
25+
class PrincipalAwsDeserializer : JsonDeserializer<List<String>>() {
26+
override fun deserialize(p: JsonParser?, ctxt: DeserializationContext?): List<String> {
27+
try {
28+
val path = p?.parsingContext?.pathAsPointer()
29+
val node = p?.codec?.readTree<JsonNode>(p)
30+
return when {
31+
node?.isNull == true -> emptyList()
32+
node?.isArray == true -> p.codec?.treeToValue(node, Array<String>::class.java)?.toList() ?: emptyList()
33+
node?.isTextual == true -> p.codec?.treeToValue(node, String::class.java)?.let { listOf(it) } ?: emptyList()
34+
else -> throw IllegalArgumentException("Principal deserialization error. Unexpected value: $node at path: $path")
35+
}
36+
} catch (e: JsonProcessingException) {
37+
throw RuntimeException("Principal deserialization error happened at: ${e.location}", e)
38+
}
39+
}
40+
}

‎storage-service/src/test/kotlin/S3GeneratePreviewTest.kt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import com.icerockdev.service.storage.preview.boundImage
1111
import com.icerockdev.service.storage.preview.loadImage
1212
import com.icerockdev.service.storage.s3.IS3Storage
1313
import com.icerockdev.service.storage.s3.S3StorageImpl
14-
import com.icerockdev.service.storage.s3.minioConfBuilder
14+
import com.icerockdev.service.storage.s3.s3Configuration
1515
import io.github.cdimascio.dotenv.dotenv
1616
import kotlinx.coroutines.runBlocking
1717
import org.junit.After
@@ -38,11 +38,11 @@ class S3GeneratePreviewTest {
3838
@Before
3939
fun init() {
4040
s3 = S3Client.builder()
41-
.serviceConfiguration(minioConfBuilder)
41+
.serviceConfiguration(s3Configuration)
4242
.credentialsProvider(
4343
StaticCredentialsProvider.create(
4444
AwsBasicCredentials.create(
45-
dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
45+
dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"]
4646
)
4747
)
4848
)
@@ -51,11 +51,11 @@ class S3GeneratePreviewTest {
5151
.build()
5252

5353
preSigner = S3Presigner.builder()
54-
.serviceConfiguration(minioConfBuilder)
54+
.serviceConfiguration(s3Configuration)
5555
.credentialsProvider(
5656
StaticCredentialsProvider.create(
5757
AwsBasicCredentials.create(
58-
dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
58+
dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"]
5959
)
6060
)
6161
)

‎storage-service/src/test/kotlin/S3StorageTest.kt‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import com.icerockdev.service.storage.exception.S3StorageException
77
import com.icerockdev.service.storage.mime.MimeTypeDetector
88
import com.icerockdev.service.storage.s3.IS3Storage
99
import com.icerockdev.service.storage.s3.S3StorageImpl
10-
import com.icerockdev.service.storage.s3.minioConfBuilder
10+
import com.icerockdev.service.storage.s3.s3Configuration
1111
import com.icerockdev.service.storage.s3.dto.FileObjectDto
1212
import com.icerockdev.service.storage.s3.policy.dto.ActionEnum
1313
import com.icerockdev.service.storage.s3.policy.dto.EffectEnum
@@ -55,11 +55,11 @@ class S3StorageTest {
5555
@Before
5656
fun init() {
5757
s3 = S3Client.builder()
58-
.serviceConfiguration(minioConfBuilder)
58+
.serviceConfiguration(s3Configuration)
5959
.credentialsProvider(
6060
StaticCredentialsProvider.create(
6161
AwsBasicCredentials.create(
62-
dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
62+
dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"]
6363
)
6464
)
6565
)
@@ -68,11 +68,11 @@ class S3StorageTest {
6868
.build()
6969

7070
preSigner = S3Presigner.builder()
71-
.serviceConfiguration(minioConfBuilder)
71+
.serviceConfiguration(s3Configuration)
7272
.credentialsProvider(
7373
StaticCredentialsProvider.create(
7474
AwsBasicCredentials.create(
75-
dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
75+
dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"]
7676
)
7777
)
7878
)
@@ -338,7 +338,7 @@ class S3StorageTest {
338338

339339
val jpgObject = storage.get(bucketName, jpgFileName)
340340

341-
assertEquals(metadata, jpgObject?.response()?.metadata())
341+
assertTrue(jpgObject?.response()?.metadata()?.entries?.containsAll(metadata.entries) ?: false)
342342

343343
// copy testing
344344
val copyFileName = storage.generateFileKey()
@@ -349,7 +349,7 @@ class S3StorageTest {
349349

350350
val copyObject = storage.get(bucketName, copyFileName)
351351

352-
assertEquals(metadata, copyObject?.response()?.metadata())
352+
assertTrue(copyObject?.response()?.metadata()?.entries?.containsAll(metadata.entries) ?: false)
353353

354354
assertTrue {
355355
storage.deleteBucketWithObjects(bucketName)

0 commit comments

Comments
 (0)