refactor(webui): Rename S3Manager to StreamFilesS3Manager; namespace stream-output AWS env vars. (#2195)

Author: junhaoliao

components/webui/server/.env

@@ -18,3 +18,7 @@ PRESTO_SCHEMA=default
 # Security
 RATE_LIMIT=1000
 
+# S3
+CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID=
+CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY=
+

components/webui/server/src/plugins/app/S3Manager/index.ts

@@ -3,6 +3,7 @@ import {
     S3Client,
 } from "@aws-sdk/client-s3";
 import {getSignedUrl} from "@aws-sdk/s3-request-presigner";
+import {AwsCredentialIdentity} from "@smithy/types";
 import {Nullable} from "@webui/common/utility-types";
 import fp from "fastify-plugin";
 
@@ -19,11 +20,17 @@ class S3Manager {
     /**
      * @param region
      * @param [profile]
+     * @param [credentials]
      */
-    constructor (region: string, profile: Nullable<string>) {
+    constructor (
+        region: string,
+        profile: Nullable<string>,
+        credentials: Nullable<AwsCredentialIdentity>
+    ) {
         this.#s3Client = new S3Client({
             region,
             ...((null !== profile) && {profile}),
+            ...((null !== credentials) && {credentials}),
         });
     }
 
@@ -60,25 +67,34 @@ class S3Manager {
 
 declare module "fastify" {
     interface FastifyInstance {
-        S3Manager?: S3Manager;
+        StreamFilesS3Manager?: S3Manager;
     }
 }
 
 export default fp(
     (fastify) => {
-        const region = settings.StreamFilesS3Region;
-        const profile = settings.StreamFilesS3Profile;
+        const region = settings.StreamFilesS3Region as Nullable<string>;
+        const profile = settings.StreamFilesS3Profile as Nullable<string>;
 
         // Only decorate if the region is set (i.e. s3 support is configured in package)
-        // Disable no-unnecessary-condition since linter doesn't understand that settings
-        // values are not hardcoded.
-        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
         if (null !== region && "" !== region) {
+            const {
+                CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID: accessKeyId,
+                CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY: secretAccessKey,
+            } = fastify.config;
+
             fastify.log.info(
                 {region, profile},
-                "Initializing S3Manager"
+                "Initializing StreamFilesS3Manager"
+            );
+            const credentials = (accessKeyId && secretAccessKey) ?
+                {accessKeyId, secretAccessKey} :
+                null;
+
+            fastify.decorate(
+                "StreamFilesS3Manager",
+                new S3Manager(region, profile, credentials)
             );
-            fastify.decorate("S3Manager", new S3Manager(region, profile));
         }
     },
 );

components/webui/server/src/plugins/external/env.ts

@@ -10,6 +10,8 @@ declare module "fastify" {
             USER: string;
             CLP_DB_USER: string;
             CLP_DB_PASS: string;
+            CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID: string;
+            CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY: string;
             PRESTO_CATALOG: string;
             PRESTO_SCHEMA: string;
             RATE_LIMIT: number;
@@ -56,6 +58,16 @@ const schema = {
             type: "string",
         },
 
+        // S3
+        CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID: {
+            type: "string",
+            default: "",
+        },
+        CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY: {
+            type: "string",
+            default: "",
+        },
+
         // Presto
         PRESTO_CATALOG: {
             type: "string",

components/webui/server/src/routes/api/stream-files/index.ts

@@ -69,8 +69,9 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => {
                 }
             }
 
-            if (fastify.hasDecorator("S3Manager") && "undefined" !== typeof fastify.S3Manager) {
-                streamMetadata.path = await fastify.S3Manager.getPreSignedUrl(
+            if (fastify.hasDecorator("StreamFilesS3Manager") &&
+                "undefined" !== typeof fastify.StreamFilesS3Manager) {
+                streamMetadata.path = await fastify.StreamFilesS3Manager.getPreSignedUrl(
                     `s3://${settings.StreamFilesS3PathPrefix}${streamMetadata.path}`
                 );
             } else {

tools/deployment/package-helm/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: "v2"
 name: "clp"
-version: "0.3.2-dev.1"
+version: "0.3.2-dev.2"
 description: "A Helm chart for CLP's (Compressed Log Processor) package deployment"
 type: "application"
 appVersion: "0.12.1-dev"

tools/deployment/package-helm/templates/webui-deployment.yaml

@@ -61,9 +61,9 @@ spec:
               value: {{ .Values.clpConfig.webui.rate_limit | quote }}
             {{- with .Values.clpConfig.stream_output.storage }}
             {{- if and (eq .type "s3") (eq .s3_config.aws_authentication.type "credentials") }}
-            - name: "AWS_ACCESS_KEY_ID"
+            - name: "CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID"
               value: {{ .s3_config.aws_authentication.credentials.access_key_id | quote }}
-            - name: "AWS_SECRET_ACCESS_KEY"
+            - name: "CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY"
               value: {{ .s3_config.aws_authentication.credentials.secret_access_key | quote }}
             {{- end }}{{/* if and (eq .type "s3")
                                   (eq .s3_config.aws_authentication.type "credentials") */}}

tools/deployment/package/docker-compose-all.yaml

@@ -366,8 +366,8 @@ services:
     <<: *service_defaults
     hostname: "webui"
     environment:
-      AWS_ACCESS_KEY_ID: "${CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID:-}"
-      AWS_SECRET_ACCESS_KEY: "${CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY:-}"
+      CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID: "${CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID:-}"
+      CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY: "${CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY:-}"
       CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}"
       CLP_DB_USER: "${CLP_DB_USER:-clp-user}"
       HOST: "0.0.0.0"