WD-696 change MinIO to RustFS as target S3, update deserialization and tests

Author: AndrewGnev

.env

@@ -3,8 +3,8 @@ APP_PORT=8080
 S3_ENDPOINT=http://127.0.0.30:9000
 S3_REGION=us-east-1
 S3_BUCKET=test-bucket
-MINIO_ACCESS_KEY=my_access_key
-MINIO_SECRET_KEY=my_secret_key
+RUSTFS_ACCESS_KEY=my_access_key
+RUSTFS_SECRET_KEY=my_secret_key
 JPG_TEST_OBJECT=100.jpg
 PNG_TEST_OBJECT=100.png
 GIF_TEST_OBJECT=100.gif

docker-compose.yaml

@@ -1,18 +1,17 @@
 version: '3'
 services:
-  minio:
-    image: minio/minio
+  rustfs:
+    image: rustfs/rustfs
     restart: always
     logging:
       driver: "json-file"
       options:
         max-size: "20m"
         max-file: "5"
-    environment:
-      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
-      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
     ports:
       - ${APP_HOST}:9000:9000
+    environment:
+      RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY}
+      RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY}
     volumes:
       - ./data:/data
-    command: 'server /data'

storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/policy/dto/Principal.kt

@@ -2,10 +2,17 @@ package com.icerockdev.service.storage.s3.policy.dto
 
 import com.fasterxml.jackson.annotation.JsonInclude
 import com.fasterxml.jackson.annotation.JsonProperty
+import com.fasterxml.jackson.core.JsonParser
+import com.fasterxml.jackson.core.JsonProcessingException
+import com.fasterxml.jackson.databind.DeserializationContext
+import com.fasterxml.jackson.databind.JsonDeserializer
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize
 
 @JsonInclude(JsonInclude.Include.NON_EMPTY)
 data class Principal(
     @JsonProperty("AWS")
+    @JsonDeserialize(using = PrincipalAwsDeserializer::class)
     val aws: List<String>,
     @JsonProperty("CanonicalUser")
     val canonicalUser: String?,
@@ -14,3 +21,20 @@ data class Principal(
     @JsonProperty("Service")
     val service: List<String> = emptyList(),
 )
+
+class PrincipalAwsDeserializer : JsonDeserializer<List<String>>() {
+    override fun deserialize(p: JsonParser?, ctxt: DeserializationContext?): List<String> {
+        try {
+            val path = p?.parsingContext?.pathAsPointer()
+            val node = p?.codec?.readTree<JsonNode>(p)
+            return when {
+                node?.isNull == true -> emptyList()
+                node?.isArray == true -> p.codec?.treeToValue(node, Array<String>::class.java)?.toList() ?: emptyList()
+                node?.isTextual == true -> p.codec?.treeToValue(node, String::class.java)?.let { listOf(it) } ?: emptyList()
+                else -> throw IllegalArgumentException("Principal deserialization error. Unexpected value: $node at path: $path")
+            }
+        } catch (e: JsonProcessingException) {
+            throw RuntimeException("Principal deserialization error happened at: ${e.location}", e)
+        }
+    }
+}

storage-service/src/test/kotlin/S3GeneratePreviewTest.kt

@@ -42,7 +42,7 @@ class S3GeneratePreviewTest {
             .credentialsProvider(
                 StaticCredentialsProvider.create(
                     AwsBasicCredentials.create(
-                        dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
+                        dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"]
                     )
                 )
             )
@@ -55,7 +55,7 @@ class S3GeneratePreviewTest {
             .credentialsProvider(
                 StaticCredentialsProvider.create(
                     AwsBasicCredentials.create(
-                        dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
+                        dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"]
                     )
                 )
             )

storage-service/src/test/kotlin/S3StorageTest.kt

@@ -59,7 +59,7 @@ class S3StorageTest {
             .credentialsProvider(
                 StaticCredentialsProvider.create(
                     AwsBasicCredentials.create(
-                        dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
+                        dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"]
                     )
                 )
             )
@@ -72,7 +72,7 @@ class S3StorageTest {
             .credentialsProvider(
                 StaticCredentialsProvider.create(
                     AwsBasicCredentials.create(
-                        dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
+                        dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"]
                     )
                 )
             )
@@ -338,7 +338,7 @@ class S3StorageTest {
 
         val jpgObject = storage.get(bucketName, jpgFileName)
 
-        assertEquals(metadata, jpgObject?.response()?.metadata())
+        assertTrue(jpgObject?.response()?.metadata()?.entries?.containsAll(metadata.entries) ?: false)
 
         // copy testing
         val copyFileName = storage.generateFileKey()
@@ -349,7 +349,7 @@ class S3StorageTest {
 
         val copyObject = storage.get(bucketName, copyFileName)
 
-        assertEquals(metadata, copyObject?.response()?.metadata())
+        assertTrue(copyObject?.response()?.metadata()?.entries?.containsAll(metadata.entries) ?: false)
 
         assertTrue {
             storage.deleteBucketWithObjects(bucketName)