Skip to content

Commit 2252177

Browse files
Brutus5000Brutus5000
committed
Allow Mod upload via S3
1 parent 1e0f51b commit 2252177

13 files changed

Lines changed: 280 additions & 22 deletions

File tree

‎build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ dependencies {
184184
implementation("org.jsoup:jsoup:1.21.1")
185185
implementation("com.github.jasminb:jsonapi-converter:0.14")
186186
implementation("commons-codec:commons-codec:1.18.0")
187+
implementation("software.amazon.awssdk:s3:2.31.68")
187188

188189
// Required library for FafTokenService approach (called by nimbus-jwt)
189190
runtimeOnly("org.bouncycastle:bcpkix-jdk15on:1.70")

‎compose.yaml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
services:
2+
# Set up the faf db
3+
faf-db:
4+
image: mariadb:11
5+
environment:
6+
MARIADB_DATABASE: faf
7+
MARIADB_USER: faf-api
8+
MARIADB_PASSWORD: banana
9+
MARIADB_ROOT_PASSWORD: banana
10+
healthcheck:
11+
test: [ "CMD", "healthcheck.sh", "--connect", "--innodb_initialized" ]
12+
interval: 10s
13+
timeout: 5s
14+
retries: 5
15+
ports:
16+
- "3306:3306"
17+
18+
faf-db-migrations:
19+
image: faforever/faf-db-migrations:v136
20+
command: migrate
21+
environment:
22+
FLYWAY_URL: jdbc:mysql://faf-db/faf?useSSL=false
23+
FLYWAY_USER: root
24+
FLYWAY_PASSWORD: banana
25+
FLYWAY_DATABASE: faf
26+
depends_on:
27+
faf-db:
28+
condition: service_healthy
29+
30+
faf-db-testdata:
31+
image: mariadb:11
32+
entrypoint: sh -c "apt-get update && apt-get install -y curl && curl -s https://raw.githubusercontent.com/FAForever/db/refs/heads/develop/test-data.sql | mariadb -h faf-db -uroot -pbanana -D faf"
33+
depends_on:
34+
faf-db-migrations:
35+
condition: service_completed_successfully
36+
37+
minio:
38+
image: docker.io/bitnami/minio:2025
39+
ports:
40+
- '9000:9000'
41+
- '9001:9001'
42+
environment:
43+
MINIO_ROOT_USER: admin
44+
MINIO_ROOT_PASSWORD: banana123
45+
MINIO_DEFAULT_BUCKETS: user-uploads
46+
MINIO_SCHEME: http
47+
healthcheck:
48+
test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/ready" ]
49+
interval: 2s
50+
timeout: 2s
51+
retries: 15
52+
53+
rabbitmq:
54+
image: rabbitmq:3.13-management
55+
environment:
56+
RABBITMQ_DEFAULT_VHOST: /faf-core
57+
RABBITMQ_DEFAULT_USER: faf-api
58+
RABBITMQ_DEFAULT_PASS: banana
59+
ports:
60+
- "5672:5672"

‎src/inttest/resources/config/application.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,11 @@ faf-api:
9999
file-url-format: "file://%s/%s"
100100
recaptcha:
101101
enabled: false
102+
s3:
103+
access-key: admin
104+
secret-key: banana123
105+
endpoint: http://minio
106+
user-upload-bucket: user-uploads
102107

103108
logging:
104109
level:

‎src/main/java/com/faforever/api/config/FafApiProperties.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ public class FafApiProperties {
4242
private Recaptcha recaptcha = new Recaptcha();
4343
private Monitoring monitoring = new Monitoring();
4444
private Coturn coturn = new Coturn();
45+
private S3 s3 = new S3();
4546

4647
@Data
4748
public static class OAuth2 {
@@ -298,4 +299,12 @@ public static class Monitoring {
298299
public static class Coturn {
299300
private int tokenLifetimeSeconds = 86400;
300301
}
302+
303+
@Data
304+
public static class S3 {
305+
private String endpoint;
306+
private String userUploadBucket;
307+
private String accessKey;
308+
private String secretKey;
309+
}
301310
}

‎src/main/java/com/faforever/api/config/S3Config.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package com.faforever.api.config;
2+
3+
import lombok.RequiredArgsConstructor;
4+
import org.springframework.context.annotation.Bean;
5+
import org.springframework.context.annotation.Configuration;
6+
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
7+
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
8+
import software.amazon.awssdk.regions.Region;
9+
import software.amazon.awssdk.services.s3.S3Client;
10+
import software.amazon.awssdk.services.s3.S3Configuration;
11+
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
12+
13+
import java.net.URI;
14+
15+
@Configuration
16+
@RequiredArgsConstructor
17+
public class S3Config {
18+
19+
private final FafApiProperties properties;
20+
21+
@Bean
22+
public S3Client s3Client() {
23+
return S3Client.builder()
24+
.endpointOverride(URI.create(properties.getS3().getEndpoint()))
25+
.region(Region.EU_CENTRAL_1) // region must be non-null but is ignored by some S3-compatible services
26+
.credentialsProvider(StaticCredentialsProvider.create(
27+
AwsBasicCredentials.create(properties.getS3().getAccessKey(), properties.getS3().getSecretKey())
28+
))
29+
.serviceConfiguration(S3Configuration.builder()
30+
.pathStyleAccessEnabled(true) // prevents putting the bucket name as subdomain
31+
.build())
32+
.build();
33+
}
34+
35+
@Bean
36+
public S3Presigner s3Presigner() {
37+
return S3Presigner.builder()
38+
.endpointOverride(URI.create(properties.getS3().getEndpoint()))
39+
.region(Region.EU_CENTRAL_1) // region must be non-null but is ignored by some S3-compatible services
40+
.credentialsProvider(StaticCredentialsProvider.create(
41+
AwsBasicCredentials.create(properties.getS3().getAccessKey(), properties.getS3().getSecretKey())
42+
))
43+
.serviceConfiguration(S3Configuration.builder()
44+
.pathStyleAccessEnabled(true) // prevents putting the bucket name as subdomain
45+
.build())
46+
.build();
47+
}
48+
}

‎src/main/java/com/faforever/api/config/security/WebSecurityConfig.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import org.springframework.security.authentication.LockedException;
1010
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
1111
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
12+
import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer;
1213
import org.springframework.security.oauth2.server.resource.web.DefaultBearerTokenResolver;
1314
import org.springframework.security.web.SecurityFilterChain;
1415
import org.springframework.security.web.authentication.AuthenticationFailureHandler;
@@ -32,8 +33,8 @@ public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Excepti
3233

3334
// @formatter:off
3435
http.csrf(csrfConfig -> csrfConfig.requireCsrfProtectionMatcher(new RequestMatcher() {
35-
private Pattern allowedMethods = Pattern.compile("^(GET|HEAD|TRACE|OPTIONS)$");
36-
private RequestMatcher matcher = new OrRequestMatcher(
36+
private final Pattern allowedMethods = Pattern.compile("^(GET|HEAD|TRACE|OPTIONS)$");
37+
private final RequestMatcher matcher = new OrRequestMatcher(
3738
new AntPathRequestMatcher("/oauth/authorize"),
3839
new AntPathRequestMatcher("/login"));
3940

@@ -43,7 +44,7 @@ public boolean matches(HttpServletRequest request) {
4344
}
4445
}));
4546
http.headers(headersConfig -> headersConfig.cacheControl().disable());
46-
http.formLogin().disable();
47+
http.formLogin(AbstractHttpConfigurer::disable);
4748
http.oauth2ResourceServer(oauth2Config -> {
4849
oauth2Config.bearerTokenResolver(bearerTokenResolver);
4950
oauth2Config.jwt(jwtConfig -> jwtConfig.jwtAuthenticationConverter(new FafAuthenticationConverter()));

‎src/main/java/com/faforever/api/mod/ModService.java‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,14 @@
3131
import org.springframework.stereotype.Service;
3232
import org.springframework.transaction.annotation.Transactional;
3333
import org.springframework.web.client.HttpClientErrorException;
34+
import software.amazon.awssdk.core.sync.ResponseTransformer;
35+
import software.amazon.awssdk.services.s3.S3Client;
36+
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
37+
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
38+
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
39+
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
40+
import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest;
41+
import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest;
3442

3543
import java.io.BufferedInputStream;
3644
import java.io.IOException;
@@ -40,11 +48,13 @@
4048
import java.nio.file.Files;
4149
import java.nio.file.Path;
4250
import java.nio.file.StandardCopyOption;
51+
import java.time.Duration;
4352
import java.util.ArrayList;
4453
import java.util.Enumeration;
4554
import java.util.List;
4655
import java.util.Optional;
4756
import java.util.Set;
57+
import java.util.UUID;
4858
import java.util.zip.ZipEntry;
4959
import java.util.zip.ZipFile;
5060

@@ -67,6 +77,55 @@ public class ModService {
6777
private final ModRepository modRepository;
6878
private final ModVersionRepository modVersionRepository;
6979
private final LicenseRepository licenseRepository;
80+
private final S3Client s3Client;
81+
private final S3Presigner s3Presigner;
82+
83+
private String getBucketKey(int userId, UUID requestId) {
84+
return "%s-mod-%s".formatted(userId, requestId);
85+
}
86+
87+
public String getPresignedS3Url(Player uploader, UUID requestId) {
88+
log.info("User {} requested presigned url for mod upload, request id {}", uploader.getId(), requestId);
89+
90+
checkUploaderVaultBan(uploader);
91+
92+
PutObjectRequest putObjectRequest = PutObjectRequest.builder()
93+
.bucket(properties.getS3().getUserUploadBucket())
94+
.key(getBucketKey(uploader.getId(), requestId))
95+
.build();
96+
97+
PutObjectPresignRequest putObjectPresignRequest = PutObjectPresignRequest.builder()
98+
.signatureDuration(Duration.ofHours(1))
99+
.putObjectRequest(putObjectRequest)
100+
.build();
101+
102+
PresignedPutObjectRequest presignedRequest = s3Presigner.presignPutObject(putObjectPresignRequest);
103+
104+
return presignedRequest.url().toString();
105+
}
106+
107+
public Path getModFromS3Location(Player uploader, UUID requestId) throws IOException {
108+
Path tempDir = Files.createTempDirectory("mod-download");
109+
String bucketKey = getBucketKey(uploader.getId(), requestId);
110+
Path tempFile = tempDir.resolve(bucketKey + ".zip");
111+
112+
checkUploaderVaultBan(uploader);
113+
114+
GetObjectRequest request = GetObjectRequest.builder()
115+
.bucket(properties.getS3().getUserUploadBucket())
116+
.key(bucketKey)
117+
.build();
118+
119+
s3Client.getObject(request, ResponseTransformer.toFile(tempFile));
120+
121+
return tempFile;
122+
}
123+
124+
public void deleteModFromS3Location(Player uploader, UUID requestId) throws IOException {
125+
String bucketKey = getBucketKey(uploader.getId(), requestId);
126+
127+
s3Client.deleteObject(DeleteObjectRequest.builder().bucket(properties.getS3().getUserUploadBucket()).key(bucketKey).build());
128+
}
70129

71130
@SneakyThrows
72131
@Transactional
@@ -248,7 +307,7 @@ private void validateModInfo(com.faforever.commons.mod.Mod modInfo) {
248307
final Integer versionInt = Ints.tryParse(modVersion.toString());
249308
if (versionInt == null) {
250309
errors.add(new Error(ErrorCode.MOD_VERSION_NOT_A_NUMBER, modVersion.toString()));
251-
} else if (!isModVersionValidRange(versionInt)){
310+
} else if (!isModVersionValidRange(versionInt)) {
252311
errors.add(new Error(ErrorCode.MOD_VERSION_INVALID_RANGE, MOD_VERSION_MIN_VALUE, MOD_VERSION_MAX_VALUE));
253312
}
254313
}

‎src/main/java/com/faforever/api/mod/ModUploadMetadata.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,10 @@
11
package com.faforever.api.mod;
22

3-
public record ModUploadMetadata(Integer licenseId, String repositoryUrl) {
3+
import java.util.UUID;
4+
5+
public record ModUploadMetadata(
6+
UUID requestId,
7+
Integer licenseId,
8+
String repositoryUrl
9+
) {
410
}

‎src/main/java/com/faforever/api/mod/ModsController.java‎

Lines changed: 47 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,75 @@
11
package com.faforever.api.mod;
22

3-
import com.faforever.api.config.FafApiProperties;
3+
import com.faforever.api.data.domain.Player;
44
import com.faforever.api.player.PlayerService;
55
import com.faforever.api.security.OAuthScope;
66
import io.swagger.v3.oas.annotations.Operation;
7-
import io.swagger.v3.oas.annotations.responses.ApiResponse;
8-
import io.swagger.v3.oas.annotations.responses.ApiResponses;
97
import lombok.RequiredArgsConstructor;
8+
import lombok.extern.slf4j.Slf4j;
109
import org.springframework.security.access.prepost.PreAuthorize;
1110
import org.springframework.security.core.Authentication;
11+
import org.springframework.web.bind.annotation.GetMapping;
12+
import org.springframework.web.bind.annotation.PostMapping;
13+
import org.springframework.web.bind.annotation.RequestBody;
1214
import org.springframework.web.bind.annotation.RequestMapping;
13-
import org.springframework.web.bind.annotation.RequestMethod;
1415
import org.springframework.web.bind.annotation.RequestParam;
1516
import org.springframework.web.bind.annotation.RequestPart;
1617
import org.springframework.web.bind.annotation.RestController;
1718
import org.springframework.web.multipart.MultipartFile;
1819

1920
import java.io.IOException;
2021
import java.nio.file.Path;
22+
import java.util.UUID;
2123

22-
import static org.springframework.http.MediaType.APPLICATION_JSON_UTF8_VALUE;
24+
import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
2325

2426
@RestController
2527
@RequestMapping(path = "/mods")
28+
@Slf4j
2629
@RequiredArgsConstructor
2730
public class ModsController {
2831

2932
private final PlayerService playerService;
3033
private final ModService modService;
31-
private final FafApiProperties fafApiProperties;
32-
33-
@Operation(summary = "Upload a mod")
34-
@ApiResponses(value = {
35-
@ApiResponse(responseCode = "200", description = "Success"),
36-
@ApiResponse(responseCode = "401", description = "Unauthorized"),
37-
@ApiResponse(responseCode = "500", description = "Failure")})
38-
@RequestMapping(path = "/upload", method = RequestMethod.POST, produces = APPLICATION_JSON_UTF8_VALUE)
34+
35+
@Operation(summary = "Begin process of uploading a mod (as a zip file)")
36+
@GetMapping(path = "/upload/start", produces = APPLICATION_JSON_VALUE)
37+
@PreAuthorize("hasScope('" + OAuthScope._UPLOAD_MOD + "')")
38+
public UploadUrlResponse startUpload(Authentication authentication) {
39+
UUID requestId = UUID.randomUUID();
40+
String presignedUrl = modService.getPresignedS3Url(playerService.getPlayer(authentication), requestId);
41+
42+
return new UploadUrlResponse(presignedUrl, requestId);
43+
}
44+
45+
@Operation(summary = "Notify about mod upload completion")
46+
@PostMapping(path = "/upload/complete", produces = APPLICATION_JSON_VALUE)
47+
@PreAuthorize("hasScope('" + OAuthScope._UPLOAD_MOD + "')")
48+
public void completeUpload(@RequestBody ModUploadMetadata metadata,
49+
Authentication authentication) throws IOException {
50+
Player uploader = playerService.getPlayer(authentication);
51+
52+
log.info("User {} reported completed mod upload, request id {}", uploader.getId(), metadata.requestId());
53+
Path tempFile = modService.getModFromS3Location(uploader, metadata.requestId());
54+
55+
try {
56+
log.debug("Process uploaded file @ {}", tempFile);
57+
modService.processUploadedMod(
58+
tempFile,
59+
tempFile.getFileName().toString(),
60+
playerService.getPlayer(authentication),
61+
metadata.licenseId(),
62+
metadata.repositoryUrl()
63+
);
64+
} finally {
65+
log.debug("Delete uploaded file for request id {}", metadata.requestId());
66+
modService.deleteModFromS3Location(uploader, metadata.requestId());
67+
}
68+
}
69+
70+
@Deprecated
71+
@Operation(summary = "Upload a mod (as a zip file)")
72+
@PostMapping(path = "/upload", produces = APPLICATION_JSON_VALUE)
3973
@PreAuthorize("hasScope('" + OAuthScope._UPLOAD_MOD + "')")
4074
public void uploadMod(
4175
@RequestParam("file") MultipartFile file,

‎src/main/java/com/faforever/api/mod/UploadUrlResponse.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package com.faforever.api.mod;
2+
3+
import java.util.UUID;
4+
5+
public record UploadUrlResponse(String uploadUrl, UUID requestId) {
6+
}

0 commit comments

Comments
 (0)