Add AWS native functionality

* Based on the configuration variable AWS_S3_DIRECT_ACCESS
  read build logs, documentation and readme files directly
  from S3, rather than the S3 web frontend.
  * Add handling of root urls to map to index.html
* Based on AWS_USE_IAM_ROLE use instance roles to access
  S3 rather than access key/secret
* Based on ENABLE_PACKAGE_UPLOAD_PRESIGNED_URLS and
  ENABLE_BUILD_LOGS_PRESIGNED_URLS, create presigned URLs
  to pass to builders for logs and docs upload.
* Add functionality to the Dockerfile to be able to build
  spi in debug vs release
* Do not add back cookies in responses, the site does not
  use cookies.
* Make the gitlab project id for build triggers configurable
  via GITLAB_PROJECT_ID
*

Author: kcieplak

Dockerfile

@@ -19,6 +19,7 @@
 # Build image
 # ================================
 FROM registry.gitlab.com/finestructure/spi-base:2.2.1 AS build
+ARG COMPILATION_MODE="release"
 
 # Set up a build area
 WORKDIR /build
@@ -39,7 +40,7 @@ RUN mkdir /staging
 
 # Build everything, with optimizations, with static linking, and using jemalloc
 # N.B.: The static version of jemalloc is incompatible with the static Swift runtime.
-RUN swift build -c release \
+RUN swift build -c ${COMPILATION_MODE} \
         --enable-experimental-prebuilts \
         --static-swift-stdlib \
         -Xlinker -ljemalloc
@@ -48,20 +49,19 @@ RUN swift build -c release \
 WORKDIR /staging
 
 # Copy main executable to staging area
-RUN cp "$(swift build --package-path /build -c release --show-bin-path)/Run" ./
+RUN cp "$(swift build --package-path /build -c $COMPILATION_MODE --show-bin-path)/Run" ./
 
 # Copy static swift backtracer binary to staging area
 RUN cp "/usr/libexec/swift/linux/swift-backtrace-static" ./
 
 # Copy resources bundled by SPM to staging area
-RUN find -L "$(swift build --package-path /build -c release --show-bin-path)/" -regex '.*\.resources$' -exec cp -Ra {} ./ \;
+RUN find -L "$(swift build --package-path /build -c $COMPILATION_MODE --show-bin-path)/" -regex '.*\.resources$' -exec cp -Ra {} ./ \;
 
 # Copy any resources from the public directory and views directory if the directories exist
 # Ensure that by default, neither the directory nor any of its contents are writable.
 RUN [ -d /build/Public ] && { mv /build/Public ./Public && chmod -R a-w ./Public; } || true
 RUN [ -d /build/Resources ] && { mv /build/Resources ./Resources && chmod -R a-w ./Resources; } || true
 
-
 # ================================
 # Run image
 # ================================
@@ -75,6 +75,9 @@ FROM registry.gitlab.com/finestructure/spi-base:2.2.1
 # Create a vapor user and group with /app as its home directory
 # RUN useradd --user-group --create-home --system --home-dir /app vapor
 
+# Download the AWS global certificate for SSL connections and add it to the cert store
+RUN curl -L -o /usr/local/share/ca-certificates/aws-global-bundle.crt https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates
+
 # Switch to the new home directory
 WORKDIR /app
 

Package.swift

@@ -79,6 +79,7 @@ let package = Package(
                     .product(name: "SwiftPMPackageCollections", package: "swift-package-manager"),
                     .product(name: "Vapor", package: "vapor"),
                     .product(name: "SotoCognitoAuthentication", package: "soto-cognito-authentication"),
+                    .product(name: "SotoS3", package: "soto"),
                     .product(name: "JWTKit", package: "jwt-kit")
                 ],
                 swiftSettings: swiftSettings,

Sources/App/Controllers/PackageController+routes.swift

@@ -154,20 +154,95 @@ enum PackageController {
             ).encodeResponse(for: req)
         }
 
+        // Remove cookies from response headers to prevent "upstream sent too big header" errors
+        var headers = req.headers
+        headers.remove(name: .cookie)
+        headers.replaceOrAdd(name: .contentType, value: route.contentType)
+
         return try await processor.processedPage.encodeResponse(
             status: .ok,
-            headers: req.headers.replacingOrAdding(name: .contentType,
-                                                   value: route.contentType),
+            headers: headers,
             for: req
         )
     }
 
     static func awsResponse(client: Client, route: DocRoute) async throws -> HTTPClient.Response {
         @Dependency(\.httpClient) var httpClient
+        @Dependency(\.environment) var environment
+        @Dependency(\.s3) var s3
+
+        // Check if direct S3 access is enabled
+        if environment.awsDirectS3Access() {
+            guard let bucket = environment.awsDocsBucket() else {
+                throw AppError.envVariableNotSet("AWS_DOCS_BUCKET")
+            }
+
+            // Construct the S3 key path - try exact path first
+            let primaryS3Key: String
+            if route.path.isEmpty {
+                primaryS3Key = "\(route.baseURL)/\(route.fragment.urlFragment)"
+            } else {
+                primaryS3Key = "\(route.baseURL)/\(route.fragment.urlFragment)/\(route.path)"
+            }
+            do {
+                let data = try await s3.fetchDocumentation(bucket, primaryS3Key)
+
+                // Create a mock HTTPClient.Response with the S3 data
+                let response = HTTPClient.Response(
+                    host: bucket,
+                    status: .ok,
+                    version: .http1_1,
+                    headers: HTTPHeaders([
+                        ("content-type", route.fragment.contentType),
+                        ("content-length", "\(data.count)")
+                    ]),
+                    body: ByteBuffer(data: data)
+                )
+                return response
+            } catch {
+                // If the primary key fails and this is a documentation/tutorials request,
+                // try appending /index.html as a fallback
+                if (route.fragment == .documentation || route.fragment == .tutorials) {
+                    let fallbackS3Key = route.path.isEmpty ?
+                        "\(route.baseURL)/index.html" :
+                        "\(route.baseURL)/\(route.fragment.urlFragment)/\(route.path)/index.html"
+                    do {
+                        let data = try await s3.fetchDocumentation(bucket, fallbackS3Key)
+
+                        let response = HTTPClient.Response(
+                            host: bucket,
+                            status: .ok,
+                            version: .http1_1,
+                            headers: HTTPHeaders([
+                                ("content-type", route.fragment.contentType),
+                                ("content-length", "\(data.count)")
+                            ]),
+                            body: ByteBuffer(data: data)
+                        )
+                        return response
+                    } catch let fallbackError {
+                        throw Abort(.notFound)
+                    }
+                } else {
+                    throw Abort(.notFound)
+                }
+            }
+        }
 
+        // Fallback to existing HTTP-based approach
         let url = try Self.awsDocumentationURL(route: route)
-        guard let response = try? await httpClient.fetchDocumentation(url) else {
-            throw Abort(.notFound)
+
+        let response: HTTPClient.Response
+        if environment.awsUseIamRole() {
+            guard let iamResponse = try? await httpClient.fetchDocumentationWithIAM(url) else {
+                throw Abort(.notFound)
+            }
+            response = iamResponse
+        } else {
+            guard let standardResponse = try? await httpClient.fetchDocumentation(url) else {
+                throw Abort(.notFound)
+            }
+            response = standardResponse
         }
         guard (200..<399).contains(response.status.code) else {
             // Convert anything that isn't a 2xx or 3xx from AWS into a 404 from us.
@@ -307,7 +382,7 @@ enum PackageController {
 
         do {
             @Dependency(\.s3) var s3
-            let readme = try await s3.fetchReadme(owner, repository)
+            let readme = try await s3.fetchReadme(owner.lowercased(), repository.lowercased())
             guard let branch = pkg.repository?.defaultBranch else {
                 return PackageReadme.View(model: .cacheLookupFailed(url: readmeHtmlUrl)).document()
             }
@@ -443,12 +518,17 @@ extension PackageController {
 extension PackageController {
     static func awsDocumentationURL(route: DocRoute) throws -> URI {
         @Dependency(\.environment) var environment
+
         guard let bucket = environment.awsDocsBucket() else {
             throw AppError.envVariableNotSet("AWS_DOCS_BUCKET")
         }
 
-        let baseURLHost = "\(bucket).s3-website.us-east-2.amazonaws.com"
-        let baseURL = "http://\(baseURLHost)/\(route.baseURL)"
+        // Use the correct region for the docs bucket
+        let region = environment.awsDocsBucketRegion() ?? environment.awsRegion() ?? "us-east-2"
+        let baseURLHost = "\(bucket).s3-website.\(region).amazonaws.com"
+        let urlScheme = "http"
+
+        let baseURL = "\(urlScheme)://\(baseURLHost)/\(route.baseURL)"
         let path = route.path
 
         switch route.fragment {

Sources/App/Core/Dependencies/EnvironmentClient.swift

@@ -29,10 +29,22 @@ struct EnvironmentClient {
     var apiSigningKey: @Sendable () -> String?
     var appVersion: @Sendable () -> String?
     var awsAccessKeyId: @Sendable () -> String?
+    var awsBuildLogsBucket: @Sendable () -> String?
+    var awsBuildLogsBucketRegion: @Sendable () -> String?
     var awsDocsBucket: @Sendable () -> String?
+    var awsDocsBucketRegion: @Sendable () -> String?
+    var awsDocsInboxBucket: @Sendable () -> String?
+    var awsDocsInboxBucketRegion: @Sendable () -> String?
     var awsReadmeBucket: @Sendable () -> String?
+    var awsReadmeBucketRegion: @Sendable () -> String?
+    var awsRegion: @Sendable () -> String?
     var awsSecretAccessKey: @Sendable () -> String?
+    var awsUseIamRole: @Sendable () -> Bool = { XCTFail("awsUseIamRole"); return false }
+    var awsDirectS3Access: @Sendable () -> Bool = { XCTFail("awsDirectS3Access"); return false }
     var builderToken: @Sendable () -> String?
+    var enableBuildLogsPreSignedURLs: @Sendable () -> Bool = { XCTFail("enableBuildLogsPreSignedURLs"); return false }
+    var enablePackageUploadPreSignedURLs: @Sendable () -> Bool = { XCTFail("enablePackageUploadPreSignedURLs"); return false }
+    var packageUploadPreSignedURLExpiration: @Sendable () -> Int = { XCTFail("packageUploadPreSignedURLExpiration"); return 86400 }
     var buildTimeout: @Sendable () -> Int = { XCTFail("buildTimeout"); return 10 }
     var buildTriggerAllowList: @Sendable () -> [Package.Id] = { XCTFail("buildTriggerAllowList"); return [] }
     var buildTriggerDownscaling: @Sendable () -> Double = { XCTFail("buildTriggerDownscaling"); return 1 }
@@ -44,6 +56,7 @@ struct EnvironmentClient {
     var gitlabApiToken: @Sendable () -> String?
     var gitlabPipelineLimit: @Sendable () -> Int = { XCTFail("gitlabPipelineLimit"); return 100 }
     var gitlabPipelineToken: @Sendable () -> String?
+    var gitlabProjectId: @Sendable () -> Int = { XCTFail("gitlabProjectId"); return 19564054 }
     var hideStagingBanner: @Sendable () -> Bool = { XCTFail("hideStagingBanner"); return Constants.defaultHideStagingBanner }
     var loadSPIManifest: @Sendable (String) -> SPIManifest.Manifest?
     var maintenanceMessage: @Sendable () -> String?
@@ -82,10 +95,32 @@ extension EnvironmentClient: DependencyKey {
             apiSigningKey: { Environment.get("API_SIGNING_KEY") },
             appVersion: { App.appVersion },
             awsAccessKeyId: { Environment.get("AWS_ACCESS_KEY_ID") },
+            awsBuildLogsBucket: { Environment.get("AWS_BUILD_LOGS_BUCKET") },
+            awsBuildLogsBucketRegion: { Environment.get("AWS_BUILD_LOGS_BUCKET_REGION") },
             awsDocsBucket: { Environment.get("AWS_DOCS_BUCKET") },
+            awsDocsBucketRegion: { Environment.get("AWS_DOCS_BUCKET_REGION") },
+            awsDocsInboxBucket: { Environment.get("AWS_DOCS_INBOX_BUCKET") },
+            awsDocsInboxBucketRegion: { Environment.get("AWS_DOCS_INBOX_BUCKET_REGION") },
             awsReadmeBucket: { Environment.get("AWS_README_BUCKET") },
+            awsReadmeBucketRegion: { Environment.get("AWS_README_BUCKET_REGION") },
+            awsRegion: { Environment.get("AWS_REGION") },
             awsSecretAccessKey: { Environment.get("AWS_SECRET_ACCESS_KEY") },
+            awsUseIamRole: {
+                Environment.get("AWS_USE_IAM_ROLE").flatMap(\.asBool) ?? false
+            },
+            awsDirectS3Access: {
+                Environment.get("AWS_S3_DIRECT_ACCESS").flatMap(\.asBool) ?? false
+            },
             builderToken: { Environment.get("BUILDER_TOKEN") },
+            enableBuildLogsPreSignedURLs: {
+                Environment.get("ENABLE_BUILD_LOGS_PRESIGNED_URLS").flatMap(\.asBool) ?? false
+            },
+            enablePackageUploadPreSignedURLs: {
+                Environment.get("ENABLE_PACKAGE_UPLOAD_PRESIGNED_URLS").flatMap(\.asBool) ?? false
+            },
+            packageUploadPreSignedURLExpiration: {
+                Environment.get("PACKAGE_UPLOAD_PRESIGNED_URL_EXPIRATION").flatMap(Int.init) ?? 86400
+            },
             buildTimeout: { Environment.get("BUILD_TIMEOUT").flatMap(Int.init) ?? 10 },
             buildTriggerAllowList: {
                 Environment.decode("BUILD_TRIGGER_ALLOW_LIST", as: [Package.Id].self) ?? []
@@ -118,6 +153,7 @@ extension EnvironmentClient: DependencyKey {
                 ?? Constants.defaultGitlabPipelineLimit
             },
             gitlabPipelineToken: { Environment.get("GITLAB_PIPELINE_TOKEN") },
+            gitlabProjectId: { Environment.get("GITLAB_PROJECT_ID").flatMap(Int.init) ?? 19564054 },
             hideStagingBanner: {
                 Environment.get("HIDE_STAGING_BANNER").flatMap(\.asBool)
                     ?? Constants.defaultHideStagingBanner

Sources/App/Core/Dependencies/HTTPClient.swift

@@ -16,6 +16,10 @@ import AsyncHTTPClient
 import Dependencies
 import DependenciesMacros
 import Vapor
+import Foundation
+import SotoS3
+import SotoCore
+import SotoSignerV4
 
 
 @DependencyClient
@@ -27,6 +31,7 @@ struct HTTPClient {
     var post: @Sendable (_ url: String, _ headers: HTTPHeaders, _ body: Data?) async throws -> Response
 
     var fetchDocumentation: @Sendable (_ url: URI) async throws -> Response
+    var fetchDocumentationWithIAM: @Sendable (_ url: URI) async throws -> Response
     var fetchHTTPStatusCode: @Sendable (_ url: String) async throws -> HTTPStatus
     var mastodonPost: @Sendable (_ message: String) async throws -> Void
     var postAnalyticsEvent: @Sendable (_ kind: Analytics.Event.Kind, _ path: Analytics.Path, _ user: User?) async throws -> Void
@@ -46,6 +51,9 @@ extension HTTPClient: DependencyKey {
             fetchDocumentation: { url in
                 try await Vapor.HTTPClient.shared.get(url: url.string).get()
             },
+            fetchDocumentationWithIAM: { url in
+                try await Self.fetchWithIAMAuth(url: url)
+            },
             fetchHTTPStatusCode: { url in
                 var config = Vapor.HTTPClient.Configuration()
                 // We're forcing HTTP/1 due to a bug in Github's HEAD request handling
@@ -104,6 +112,84 @@ private let globallySharedHTTPClient: Vapor.HTTPClient = {
 }()
 
 
+// MARK: - AWS IAM Authentication
+extension HTTPClient {
+    static func fetchWithIAMAuth(url: URI) async throws -> Response {
+        // Create AWS client with default credential provider
+        let awsClient = AWSClient(
+            credentialProvider: .default,
+            httpClientProvider: .createNew
+        )
+
+        defer {
+            Task {
+                try await awsClient.shutdown()
+            }
+        }
+
+        // Extract region from URL
+        let urlComponents = URLComponents(string: url.string)
+        guard let host = urlComponents?.host else {
+            throw AppError.genericError(nil, "Invalid URL: \(url.string)")
+        }
+
+        let region = extractRegionFromHost(host) ?? .useast2
+
+        // Get credentials from the credential provider
+        let credentials: Credential
+        do {
+            credentials = try await awsClient.credentialProvider.getCredential(
+                on: awsClient.eventLoopGroup.next(),
+                logger: Logger(label: "aws-credentials")
+            ).get()
+        } catch {
+            throw AppError.genericError(nil, "Failed to retrieve AWS credentials: \(error)")
+        }
+
+        // Use Soto's AWSSigner to sign the request headers
+        let signer = AWSSigner(
+            credentials: credentials,
+            name: "execute-api",
+            region: region.rawValue
+        )
+
+        // Create the request URL
+        guard let requestURL = URL(string: url.string) else {
+            throw AppError.genericError(nil, "Invalid URL: \(url.string)")
+        }
+
+        // Sign the headers for the request
+        let signedHeaders: HTTPHeaders
+        do {
+            signedHeaders = try await signer.signHeaders(
+                url: requestURL,
+                method: HTTPMethod.GET,
+                headers: HTTPHeaders(),
+                body: nil
+            )
+        } catch {
+            throw AppError.genericError(nil, "Failed to sign request: \(error)")
+        }
+
+        // Create request with signed headers
+        let request = try Request(url: url.string, method: .GET, headers: signedHeaders)
+
+        // Execute the request
+        return try await shared.execute(request: request).get()
+    }
+
+    private static func extractRegionFromHost(_ host: String) -> Region? {
+        // Extract region from API Gateway URL format: {api-id}.execute-api.{region}.amazonaws.com
+        let components = host.split(separator: ".")
+        if components.count >= 4 && components[1] == "execute-api" && components[3] == "amazonaws" {
+            let regionString = String(components[2])
+            return Region(rawValue: regionString)
+        }
+        return nil
+    }
+}
+
+
 #if DEBUG
 // Convenience initialisers to make testing easier