Allow Mod upload via S3

Author: Brutus5000

build.gradle

@@ -184,6 +184,7 @@ dependencies {
   implementation("org.jsoup:jsoup:1.21.1")
   implementation("com.github.jasminb:jsonapi-converter:0.14")
   implementation("commons-codec:commons-codec:1.18.0")
+  implementation("software.amazon.awssdk:s3:2.31.68")
 
   // Required library for FafTokenService approach (called by nimbus-jwt)
   runtimeOnly("org.bouncycastle:bcpkix-jdk15on:1.70")

compose.yaml

@@ -0,0 +1,60 @@
+services:
+  # Set up the faf db
+  faf-db:
+    image: mariadb:11
+    environment:
+      MARIADB_DATABASE: faf
+      MARIADB_USER: faf-api
+      MARIADB_PASSWORD: banana
+      MARIADB_ROOT_PASSWORD: banana
+    healthcheck:
+      test: [ "CMD", "healthcheck.sh", "--connect", "--innodb_initialized" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    ports:
+      - "3306:3306"
+
+  faf-db-migrations:
+    image: faforever/faf-db-migrations:v136
+    command: migrate
+    environment:
+      FLYWAY_URL: jdbc:mysql://faf-db/faf?useSSL=false
+      FLYWAY_USER: root
+      FLYWAY_PASSWORD: banana
+      FLYWAY_DATABASE: faf
+    depends_on:
+      faf-db:
+        condition: service_healthy
+
+  faf-db-testdata:
+    image: mariadb:11
+    entrypoint: sh -c "apt-get update && apt-get install -y curl && curl -s https://raw.githubusercontent.com/FAForever/db/refs/heads/develop/test-data.sql | mariadb -h faf-db -uroot -pbanana -D faf"
+    depends_on:
+      faf-db-migrations:
+        condition: service_completed_successfully
+
+  minio:
+    image: docker.io/bitnami/minio:2025
+    ports:
+      - '9000:9000'
+      - '9001:9001'
+    environment:
+      MINIO_ROOT_USER: admin
+      MINIO_ROOT_PASSWORD: banana123
+      MINIO_DEFAULT_BUCKETS: user-uploads
+      MINIO_SCHEME: http
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/ready" ]
+      interval: 2s
+      timeout: 2s
+      retries: 15
+
+  rabbitmq:
+    image: rabbitmq:3.13-management
+    environment:
+      RABBITMQ_DEFAULT_VHOST: /faf-core
+      RABBITMQ_DEFAULT_USER: faf-api
+      RABBITMQ_DEFAULT_PASS: banana
+    ports:
+      - "5672:5672"

src/main/java/com/faforever/api/config/FafApiProperties.java

@@ -42,6 +42,7 @@ public class FafApiProperties {
   private Recaptcha recaptcha = new Recaptcha();
   private Monitoring monitoring = new Monitoring();
   private Coturn coturn = new Coturn();
+  private S3 s3 = new S3();
 
   @Data
   public static class OAuth2 {
@@ -298,4 +299,12 @@ public static class Monitoring {
   public static class Coturn {
     private int tokenLifetimeSeconds = 86400;
   }
+
+  @Data
+  public static class S3 {
+    private String endpoint;
+    private String userUploadBucket;
+    private String accessKey;
+    private String secretKey;
+  }
 }

src/main/java/com/faforever/api/config/S3Config.java

@@ -0,0 +1,48 @@
+package com.faforever.api.config;
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3Configuration;
+import software.amazon.awssdk.services.s3.presigner.S3Presigner;
+
+import java.net.URI;
+
+@Configuration
+@RequiredArgsConstructor
+public class S3Config {
+
+  private final FafApiProperties properties;
+
+  @Bean
+  public S3Client s3Client() {
+    return S3Client.builder()
+      .endpointOverride(URI.create(properties.getS3().getEndpoint()))
+      .region(Region.EU_CENTRAL_1) // region must be non-null but is ignored by some S3-compatible services
+      .credentialsProvider(StaticCredentialsProvider.create(
+        AwsBasicCredentials.create(properties.getS3().getAccessKey(), properties.getS3().getSecretKey())
+      ))
+      .serviceConfiguration(S3Configuration.builder()
+        .pathStyleAccessEnabled(true) // prevents putting the bucket name as subdomain
+        .build())
+      .build();
+  }
+
+  @Bean
+  public S3Presigner s3Presigner() {
+    return S3Presigner.builder()
+      .endpointOverride(URI.create(properties.getS3().getEndpoint()))
+      .region(Region.EU_CENTRAL_1) // region must be non-null but is ignored by some S3-compatible services
+      .credentialsProvider(StaticCredentialsProvider.create(
+        AwsBasicCredentials.create(properties.getS3().getAccessKey(), properties.getS3().getSecretKey())
+      ))
+      .serviceConfiguration(S3Configuration.builder()
+        .pathStyleAccessEnabled(true) // prevents putting the bucket name as subdomain
+        .build())
+      .build();
+  }
+}

src/main/java/com/faforever/api/mod/ModService.java

@@ -31,6 +31,14 @@
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.client.HttpClientErrorException;
+import software.amazon.awssdk.core.sync.ResponseTransformer;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.presigner.S3Presigner;
+import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest;
+import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest;
 
 import java.io.BufferedInputStream;
 import java.io.IOException;
@@ -40,11 +48,13 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.UUID;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
@@ -67,6 +77,55 @@ public class ModService {
   private final ModRepository modRepository;
   private final ModVersionRepository modVersionRepository;
   private final LicenseRepository licenseRepository;
+  private final S3Client s3Client;
+  private final S3Presigner s3Presigner;
+
+  private String getBucketKey(int userId, UUID requestId) {
+    return "%s-mod-%s".formatted(userId, requestId);
+  }
+
+  public String getPresignedS3Url(Player uploader, UUID requestId) {
+    log.info("User {} requested presigned url for mod upload, request id {}", uploader.getId(), requestId);
+
+    checkUploaderVaultBan(uploader);
+
+    PutObjectRequest putObjectRequest = PutObjectRequest.builder()
+      .bucket(properties.getS3().getUserUploadBucket())
+      .key(getBucketKey(uploader.getId(), requestId))
+      .build();
+
+    PutObjectPresignRequest putObjectPresignRequest = PutObjectPresignRequest.builder()
+      .signatureDuration(Duration.ofHours(1))
+      .putObjectRequest(putObjectRequest)
+      .build();
+
+    PresignedPutObjectRequest presignedRequest = s3Presigner.presignPutObject(putObjectPresignRequest);
+
+    return presignedRequest.url().toString();
+  }
+
+  public Path getModFromS3Location(Player uploader, UUID requestId) throws IOException {
+    Path tempDir = Files.createTempDirectory("mod-download");
+    String bucketKey = getBucketKey(uploader.getId(), requestId);
+    Path tempFile = tempDir.resolve(bucketKey + ".zip");
+
+    checkUploaderVaultBan(uploader);
+
+    GetObjectRequest request = GetObjectRequest.builder()
+      .bucket(properties.getS3().getUserUploadBucket())
+      .key(bucketKey)
+      .build();
+
+    s3Client.getObject(request, ResponseTransformer.toFile(tempFile));
+
+    return tempFile;
+  }
+
+  public void deleteModFromS3Location(Player uploader, UUID requestId) throws IOException {
+    String bucketKey = getBucketKey(uploader.getId(), requestId);
+
+    s3Client.deleteObject(DeleteObjectRequest.builder().bucket(properties.getS3().getUserUploadBucket()).key(bucketKey).build());
+  }
 
   @SneakyThrows
   @Transactional
@@ -94,8 +153,7 @@ public void processUploadedMod(Path uploadedFile, String originalFilename, Playe
     short version = (short) Integer.parseInt(modInfo.getVersion().toString());
 
     if (!canUploadMod(displayName, uploader)) {
-      Mod mod = modRepository.findOneByDisplayName(displayName)
-        .orElseThrow(() -> new IllegalStateException("Mod could not be found"));
+      Mod mod = modRepository.findOneByDisplayName(displayName).orElseThrow(() -> new IllegalStateException("Mod could not be found"));
       throw new ApiException(new Error(ErrorCode.MOD_NOT_ORIGINAL_AUTHOR, mod.getAuthor(), displayName));
     }
 
@@ -159,9 +217,7 @@ private void validateZipFileSafety(Path uploadedFile) {
 
     try {
       // Unzipping directory already invokes the checks we want to perform
-      Unzipper.from(uploadedFile)
-        .to(tempDirectory)
-        .unzip();
+      Unzipper.from(uploadedFile).to(tempDirectory).unzip();
 
     } finally {
       log.debug("Delete unzipped files in folder {}", tempDirectory);
@@ -188,11 +244,7 @@ private void validateModStructure(Path uploadedFile) {
   }
 
   private boolean modExists(String displayName, short version) {
-    ModVersion probe = new ModVersion()
-      .setVersion(version)
-      .setMod(new Mod()
-        .setDisplayName(displayName)
-      );
+    ModVersion probe = new ModVersion().setVersion(version).setMod(new Mod().setDisplayName(displayName));
 
     return modVersionRepository.exists(Example.of(probe, ExampleMatcher.matching()));
   }
@@ -202,14 +254,10 @@ private boolean modUidExists(String uuid) {
   }
 
   private void checkUploaderVaultBan(Player uploader) {
-    uploader.getActiveBanOf(BanLevel.VAULT)
-      .ifPresent((banInfo) -> {
-        String message = banInfo.getDuration() == BanDurationType.PERMANENT ?
-          "You are permanently banned from uploading mods to the vault." :
-          format("You are banned from uploading mods to the vault until {0}.", banInfo.getExpiresAt());
-        throw HttpClientErrorException.create(message, HttpStatus.FORBIDDEN, "Upload forbidden",
-          HttpHeaders.EMPTY, null, null);
-      });
+    uploader.getActiveBanOf(BanLevel.VAULT).ifPresent((banInfo) -> {
+      String message = banInfo.getDuration() == BanDurationType.PERMANENT ? "You are permanently banned from uploading mods to the vault." : format("You are banned from uploading mods to the vault until {0}.", banInfo.getExpiresAt());
+      throw HttpClientErrorException.create(message, HttpStatus.FORBIDDEN, "Upload forbidden", HttpHeaders.EMPTY, null, null);
+    });
   }
 
   private boolean canUploadMod(String displayName, Player uploader) {
@@ -248,7 +296,7 @@ private void validateModInfo(com.faforever.commons.mod.Mod modInfo) {
       final Integer versionInt = Ints.tryParse(modVersion.toString());
       if (versionInt == null) {
         errors.add(new Error(ErrorCode.MOD_VERSION_NOT_A_NUMBER, modVersion.toString()));
-      } else if (!isModVersionValidRange(versionInt)){
+      } else if (!isModVersionValidRange(versionInt)) {
         errors.add(new Error(ErrorCode.MOD_VERSION_INVALID_RANGE, MOD_VERSION_MIN_VALUE, MOD_VERSION_MAX_VALUE));
       }
     }
@@ -307,22 +355,10 @@ private String generateFolderName(String displayName, short version) {
   }
 
   private void store(com.faforever.commons.mod.Mod modInfo, Optional<Path> thumbnailPath, Player uploader, String zipFileName, Integer licenseId, String repositoryUrl) {
-    ModVersion modVersion = new ModVersion()
-      .setUid(modInfo.getUid())
-      .setType(modInfo.isUiOnly() ? ModType.UI : ModType.SIM)
-      .setDescription(modInfo.getDescription())
-      .setVersion((short) Integer.parseInt(modInfo.getVersion().toString()))
-      .setFilename(MOD_PATH_PREFIX + zipFileName)
-      .setIcon(thumbnailPath.map(path -> path.getFileName().toString()).orElse(null));
+    ModVersion modVersion = new ModVersion().setUid(modInfo.getUid()).setType(modInfo.isUiOnly() ? ModType.UI : ModType.SIM).setDescription(modInfo.getDescription()).setVersion((short) Integer.parseInt(modInfo.getVersion().toString())).setFilename(MOD_PATH_PREFIX + zipFileName).setIcon(thumbnailPath.map(path -> path.getFileName().toString()).orElse(null));
 
     License newLicense = getLicenseOrDefault(licenseId);
-    Mod mod = modRepository.findOneByDisplayName(modInfo.getName())
-      .orElse(new Mod()
-        .setAuthor(modInfo.getAuthor())
-        .setDisplayName(modInfo.getName())
-        .setUploader(uploader)
-        .setLicense(newLicense)
-        .setRecommended(false));
+    Mod mod = modRepository.findOneByDisplayName(modInfo.getName()).orElse(new Mod().setAuthor(modInfo.getAuthor()).setDisplayName(modInfo.getName()).setUploader(uploader).setLicense(newLicense).setRecommended(false));
 
     if (newLicense.isLessPermissiveThan(mod.getLicense())) {
       throw ApiException.of(ErrorCode.LESS_PERMISSIVE_LICENSE);
@@ -336,8 +372,6 @@ private void store(com.faforever.commons.mod.Mod modInfo, Optional<Path> thumbna
   }
 
   public License getLicenseOrDefault(Integer licenseId) {
-    return Optional.ofNullable(licenseId)
-      .flatMap(licenseRepository::findById)
-      .orElseGet(() -> licenseRepository.getReferenceById(properties.getMod().getDefaultLicenseId()));
+    return Optional.ofNullable(licenseId).flatMap(licenseRepository::findById).orElseGet(() -> licenseRepository.getReferenceById(properties.getMod().getDefaultLicenseId()));
   }
 }

src/main/java/com/faforever/api/mod/ModUploadMetadata.java

@@ -1,4 +1,10 @@
 package com.faforever.api.mod;
 
-public record ModUploadMetadata(Integer licenseId, String repositoryUrl) {
+import java.util.UUID;
+
+public record ModUploadMetadata(
+  UUID requestId,
+  Integer licenseId,
+  String repositoryUrl
+) {
 }