test(backend): more backend testing for file sharing (#4043)

resolves #3991

- Add public policy to testcontainer bucket, so tagged files get made
public.
- Add a new test, that checks whether approved files are actually
public.
- Add a new test that checks whether the presigned URLs returned by
'extract-...-data' are accessible.

🚀 Preview: Add `preview` label to enable

Author: fhennig

backend/src/main/kotlin/org/loculus/backend/api/FileTypes.kt

@@ -21,3 +21,8 @@ data class FileIdAndUrl(
 data class FileIdAndName(val fileId: FileId, val name: String)
 
 data class FileIdAndNameAndUrl(val fileId: FileId, val name: String, val url: String)
+
+/**
+ * Strip the URL from the object.
+ */
+fun FileIdAndNameAndUrl.toFileIdAndName(): FileIdAndName = FileIdAndName(fileId, name)

backend/src/main/kotlin/org/loculus/backend/service/files/S3Service.kt

@@ -51,6 +51,10 @@ class S3Service(private val s3Config: S3Config) {
         return "${config.endpoint}/${config.bucket}/${getFileName(fileId)}"
     }
 
+    /**
+     * Sets the 'public=true' tag on the given file ID.
+     * The bucket should have a policy that files with this tag are publicly accessible.
+     */
     fun setFileToPublic(fileId: FileId) {
         val config = getS3BucketConfig()
         getInternalClient().setObjectTags(

backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt

@@ -2,6 +2,7 @@ package org.loculus.backend.controller
 
 import io.minio.MakeBucketArgs
 import io.minio.MinioClient
+import io.minio.SetBucketPolicyArgs
 import mu.KotlinLogging
 import org.junit.jupiter.api.extension.BeforeEachCallback
 import org.junit.jupiter.api.extension.ExtendWith
@@ -254,4 +255,24 @@ private fun createBucket(container: MinIOContainer, region: String, bucket: Stri
             .bucket(bucket)
             .build(),
     )
+    val policy = """
+    {
+      "Version":"2012-10-17",
+      "Statement":[
+        {
+          "Effect":"Allow",
+          "Principal":"*",
+          "Action":"s3:GetObject",
+          "Resource":["arn:aws:s3:::$bucket/*"],
+          "Condition":{
+            "StringEquals":{
+              "s3:ExistingObjectTag/public":"true"
+            }
+          }
+        }
+      ]
+    }
+    """.trimIndent()
+
+    minioClient.setBucketPolicy(SetBucketPolicyArgs.builder().bucket(bucket).region(region).config(policy).build())
 }

backend/src/test/kotlin/org/loculus/backend/controller/files/FilesClient.kt

@@ -1,6 +1,7 @@
 package org.loculus.backend.controller.files
 
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import org.loculus.backend.api.FileIdAndUrl
 import org.loculus.backend.controller.jwtForDefaultUser
 import org.loculus.backend.controller.withAuth
 import org.loculus.backend.service.files.FileId
@@ -27,3 +28,11 @@ fun ResultActions.andGetFileIds(): List<FileId> = andReturn()
         val responseJson = jacksonObjectMapper().readTree(it)
         responseJson.map { UUID.fromString(it.get("fileId").textValue()) }
     }
+
+fun ResultActions.andGetFileIdsAndUrls(): List<FileIdAndUrl> = andReturn()
+    .response
+    .contentAsString
+    .let {
+        val responseJson = jacksonObjectMapper().readTree(it)
+        responseJson.map { FileIdAndUrl(UUID.fromString(it.get("fileId").textValue()), it.get("url").textValue()) }
+    }

backend/src/test/kotlin/org/loculus/backend/controller/submission/ApproveProcessedDataEndpointTest.kt

@@ -1,5 +1,6 @@
 package org.loculus.backend.controller.submission
 
+import com.fasterxml.jackson.databind.ObjectMapper
 import org.hamcrest.MatcherAssert.assertThat
 import org.hamcrest.Matchers.containsString
 import org.hamcrest.Matchers.hasSize
@@ -11,11 +12,15 @@ import org.loculus.backend.api.ApproveDataScope.WITHOUT_WARNINGS
 import org.loculus.backend.api.Status.APPROVED_FOR_RELEASE
 import org.loculus.backend.api.Status.IN_PROCESSING
 import org.loculus.backend.api.Status.PROCESSED
+import org.loculus.backend.api.SubmittedProcessedData
+import org.loculus.backend.api.toFileIdAndName
+import org.loculus.backend.config.BackendSpringProperty
 import org.loculus.backend.controller.ALTERNATIVE_DEFAULT_USER_NAME
 import org.loculus.backend.controller.DEFAULT_ORGANISM
 import org.loculus.backend.controller.DEFAULT_USER_NAME
 import org.loculus.backend.controller.EndpointTest
 import org.loculus.backend.controller.OTHER_ORGANISM
+import org.loculus.backend.controller.S3_CONFIG
 import org.loculus.backend.controller.assertHasError
 import org.loculus.backend.controller.assertStatusIs
 import org.loculus.backend.controller.expectUnauthorizedResponse
@@ -29,11 +34,18 @@ import org.springframework.http.MediaType
 import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content
 import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath
 import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status
-
-@EndpointTest
+import java.net.URI
+import java.net.http.HttpClient
+import java.net.http.HttpRequest
+import java.net.http.HttpResponse
+
+@EndpointTest(
+    properties = ["${BackendSpringProperty.BACKEND_CONFIG_PATH}=$S3_CONFIG" ],
+)
 class ApproveProcessedDataEndpointTest(
     @Autowired val client: SubmissionControllerClient,
     @Autowired val convenienceClient: SubmissionConvenienceClient,
+    @Autowired val objectMapper: ObjectMapper,
 ) {
 
     @Test
@@ -404,4 +416,37 @@ class ApproveProcessedDataEndpointTest(
 
         convenienceClient.getSequenceEntry(accessionVersions.first()).assertStatusIs(APPROVED_FOR_RELEASE)
     }
+
+    @Test
+    fun `WHEN entries with extra files are approved THEN the files become public`() {
+        convenienceClient.submitDefaultFiles(includeFileMapping = true)
+        val unprocessedData = convenienceClient.extractUnprocessedData()
+        // create submittable data with the file IDs included
+        val submittableData: List<SubmittedProcessedData> = unprocessedData.map {
+            SubmittedProcessedData(
+                accession = it.accession,
+                version = it.version,
+                data = defaultProcessedData.copy(
+                    files = it.data.files!!.map {
+                        it.key to it.value.map { x -> x.toFileIdAndName() }
+                    }.toMap(),
+                ),
+            )
+        }
+        convenienceClient.submitProcessedData(submittableData)
+        client.approveProcessedSequenceEntries(ALL)
+        val releasedData = convenienceClient.getReleasedData()
+
+        val tree = objectMapper.readTree(releasedData[0].metadata["fileField"]!!.asText())
+
+        val fileUrl = tree.get(0).get("url").asText()
+
+        val client = HttpClient.newHttpClient()
+        val request = HttpRequest.newBuilder()
+            .uri(URI.create(fileUrl))
+            .build()
+
+        val response = client.send(request, HttpResponse.BodyHandlers.ofString())
+        assertThat(response.statusCode(), `is`(200))
+    }
 }

backend/src/test/kotlin/org/loculus/backend/controller/submission/ExtractUnprocessedDataEndpointTest.kt

@@ -42,6 +42,10 @@ import org.springframework.http.HttpHeaders.ETAG
 import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header
 import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath
 import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status
+import java.net.URI
+import java.net.http.HttpClient
+import java.net.http.HttpRequest
+import java.net.http.HttpResponse
 
 @EndpointTest(
     properties = [
@@ -254,4 +258,24 @@ class ExtractUnprocessedDataEndpointTest(
             ),
         )
     }
+
+    @Test
+    fun `GIVEN returns file mapping THEN the URLs are valid`() {
+        convenienceClient.submitDefaultFiles(includeFileMapping = true)
+
+        val result = client.extractUnprocessedData(
+            numberOfSequenceEntries = DefaultFiles.NUMBER_OF_SEQUENCES,
+        )
+        val responseBody = result.expectNdjsonAndGetContent<UnprocessedData>()
+        val url = responseBody.first().data.files!!["fileField"]!!.first().url
+
+        val client = HttpClient.newHttpClient()
+        val request = HttpRequest.newBuilder()
+            .uri(URI.create(url))
+            .build()
+
+        val response = client.send(request, HttpResponse.BodyHandlers.ofString())
+        assertThat(response.statusCode(), `is`(200))
+        assertThat(response.body(), `is`("Hello, world!"))
+    }
 }

backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt

@@ -31,7 +31,7 @@ import org.loculus.backend.controller.ORGANISM_WITHOUT_CONSENSUS_SEQUENCES
 import org.loculus.backend.controller.OTHER_ORGANISM
 import org.loculus.backend.controller.expectNdjsonAndGetContent
 import org.loculus.backend.controller.files.FilesClient
-import org.loculus.backend.controller.files.andGetFileIds
+import org.loculus.backend.controller.files.andGetFileIdsAndUrls
 import org.loculus.backend.controller.generateJwtFor
 import org.loculus.backend.controller.getAccessionVersions
 import org.loculus.backend.controller.groupmanagement.GroupManagementControllerClient
@@ -43,6 +43,10 @@ import org.springframework.http.MediaType
 import org.springframework.test.web.servlet.ResultActions
 import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content
 import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status
+import java.net.URI
+import java.net.http.HttpClient
+import java.net.http.HttpRequest
+import java.net.http.HttpResponse
 
 data class SubmissionResult(val submissionIdMappings: List<SubmissionIdMapping>, val groupId: Int)
 
@@ -79,9 +83,28 @@ class SubmissionConvenienceClient(
 
         var fileMapping: SubmissionIdFilesMap? = null
         if (includeFileMapping) {
-            val fileId = filesClient.requestUploads(groupIdToSubmitFor, jwt = jwt).andGetFileIds()[0]
-            fileMapping = DefaultFiles.submissionIds.associateWith {
-                mapOf("fileField" to listOf(FileIdAndName(fileId, "foo.txt")))
+            val fileIdsAndUrls = filesClient.requestUploads(
+                groupIdToSubmitFor,
+                DefaultFiles.submissionIds.size,
+                jwt = jwt,
+            ).andGetFileIdsAndUrls()
+
+            fileMapping = mutableMapOf()
+
+            val client = HttpClient.newBuilder().build()
+            val fileContent = "Hello, world!".toByteArray()
+
+            DefaultFiles.submissionIds.forEachIndexed { i, submissionId ->
+
+                val request = HttpRequest.newBuilder()
+                    .uri(URI.create(fileIdsAndUrls[i].url))
+                    .PUT(HttpRequest.BodyPublishers.ofByteArray(fileContent))
+                    .build()
+
+                client.send(request, HttpResponse.BodyHandlers.ofString())
+
+                fileMapping[submissionId] =
+                    mapOf("fileField" to listOf(FileIdAndName(fileIdsAndUrls[i].fileId, "hello.txt")))
             }
         }
 
@@ -112,8 +135,15 @@ class SubmissionConvenienceClient(
         username: String = DEFAULT_USER_NAME,
         groupId: Int? = null,
         dataUseTerms: DataUseTerms = DataUseTerms.Open,
+        includeFileMapping: Boolean = false,
     ): List<AccessionVersionInterface> {
-        submitDefaultFiles(organism = organism, username = username, groupId = groupId, dataUseTerms = dataUseTerms)
+        submitDefaultFiles(
+            organism = organism,
+            username = username,
+            groupId = groupId,
+            dataUseTerms = dataUseTerms,
+            includeFileMapping = includeFileMapping,
+        )
         return extractUnprocessedData(organism = organism).getAccessionVersions()
     }